Skip to content

Commit 8a2328f

Browse files
committed
feat: add JSONL (JSON Lines) output format
Adds a jsonl_output option that writes one self-contained JSON object per line (newline-delimited JSON) instead of maintaining a single JSON array. Unlike json_output, JSONL writes are pure appends with no seek/rewrite of the existing file, which makes it safe for streaming, log shipping and tools like jq that consume newline-delimited JSON. New options: - jsonl_output: bool = False - jsonl_path: str = 'tamga.jsonl' - max_jsonl_size_mb: int = 10 Wired into log(), flush() and the buffered write path, mirroring the existing json_output implementation.
1 parent 29b9f09 commit 8a2328f

1 file changed

Lines changed: 59 additions & 0 deletions

File tree

‎tamga/main.py‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ class Tamga:
3030
"colored_output",
3131
"file_output",
3232
"json_output",
33+
"jsonl_output",
3334
"mongo_output",
3435
"sql_output",
3536
# Display settings
@@ -39,6 +40,7 @@ class Tamga:
3940
# File paths and configurations
4041
"file_path",
4142
"json_path",
43+
"jsonl_path",
4244
"sql_path",
4345
"sql_table_name",
4446
# MongoDB configuration
@@ -53,6 +55,7 @@ class Tamga:
5355
# Size limits and buffering
5456
"max_file_size_mb",
5557
"max_json_size_mb",
58+
"max_jsonl_size_mb",
5659
"max_sql_size_mb",
5760
"enable_backup",
5861
"buffer_size",
@@ -64,6 +67,7 @@ class Tamga:
6467
"_notify_executor",
6568
"_file_buffer",
6669
"_json_buffer",
70+
"_jsonl_buffer",
6771
"_buffer_lock",
6872
"_color_cache",
6973
"_json_file_handle",
@@ -77,6 +81,7 @@ def __init__(
7781
colored_output: bool = True,
7882
file_output: bool = False,
7983
json_output: bool = False,
84+
jsonl_output: bool = False,
8085
mongo_output: bool = False,
8186
sql_output: bool = False,
8287
# Display settings
@@ -86,6 +91,7 @@ def __init__(
8691
# File paths and configurations
8792
file_path: str = "tamga.log",
8893
json_path: str = "tamga.json",
94+
jsonl_path: str = "tamga.jsonl",
8995
sql_path: str = "tamga.db",
9096
sql_table_name: str = "logs",
9197
# MongoDB configuration
@@ -100,6 +106,7 @@ def __init__(
100106
# Size limits and buffering
101107
max_file_size_mb: int = 10,
102108
max_json_size_mb: int = 10,
109+
max_jsonl_size_mb: int = 10,
103110
max_sql_size_mb: int = 50,
104111
enable_backup: bool = True,
105112
buffer_size: int = 50,
@@ -139,6 +146,7 @@ def __init__(
139146
self.colored_output = colored_output
140147
self.file_output = file_output
141148
self.json_output = json_output
149+
self.jsonl_output = jsonl_output
142150
self.mongo_output = mongo_output
143151
self.sql_output = sql_output
144152

@@ -150,6 +158,7 @@ def __init__(
150158
# File paths and configurations
151159
self.file_path = file_path
152160
self.json_path = json_path
161+
self.jsonl_path = jsonl_path
153162
self.sql_path = sql_path
154163
self.sql_table_name = sql_table_name
155164

@@ -167,6 +176,7 @@ def __init__(
167176
# Size limits and buffering
168177
self.max_file_size_mb = max_file_size_mb
169178
self.max_json_size_mb = max_json_size_mb
179+
self.max_jsonl_size_mb = max_jsonl_size_mb
170180
self.max_sql_size_mb = max_sql_size_mb
171181
self.enable_backup = enable_backup
172182
self.buffer_size = buffer_size
@@ -180,6 +190,7 @@ def __init__(
180190
self._notify_executor = None
181191
self._file_buffer = []
182192
self._json_buffer = []
193+
self._jsonl_buffer = []
183194
self._buffer_lock = threading.Lock()
184195
self._color_cache = {}
185196
self._json_file_handle = None
@@ -406,6 +417,9 @@ def log(self, message: str, level: str, color: str) -> None:
406417
if self.json_output:
407418
self._buffer_json_write(log_data)
408419

420+
if self.jsonl_output:
421+
self._buffer_jsonl_write(log_data)
422+
409423
if self.sql_output:
410424
self._write_to_sql(log_data)
411425

@@ -432,6 +446,49 @@ def _buffer_json_write(self, log_data: Dict[str, Any]):
432446
if len(self._json_buffer) >= self.buffer_size:
433447
self._flush_json_buffer()
434448

449+
def _buffer_jsonl_write(self, log_data: Dict[str, Any]):
450+
"""Buffer JSON Lines writes for better performance."""
451+
with self._buffer_lock:
452+
self._jsonl_buffer.append(log_data)
453+
if len(self._jsonl_buffer) >= self.buffer_size:
454+
self._flush_jsonl_buffer()
455+
456+
def _flush_jsonl_buffer(self):
457+
"""Flush JSON Lines buffer to disk.
458+
459+
Unlike the JSON array format, JSONL appends one self-contained JSON
460+
object per line, so writes are pure appends with no seek/rewrite of the
461+
existing file. This makes it safe for streaming, log shipping and tools
462+
like `jq` that consume newline-delimited JSON.
463+
"""
464+
if not self._jsonl_buffer:
465+
return
466+
467+
self._handle_file_rotation(self.jsonl_path, self.max_jsonl_size_mb)
468+
469+
try:
470+
with open(self.jsonl_path, "a", encoding="utf-8") as f:
471+
for log in self._jsonl_buffer:
472+
f.write(
473+
json.dumps(
474+
{
475+
"level": log["level"],
476+
"message": log["base_message"],
477+
"data": log["data"],
478+
"date": log["date"],
479+
"time": log["time"],
480+
"timezone": log["timezone"],
481+
"timestamp": log["unix_timestamp"],
482+
},
483+
ensure_ascii=False,
484+
separators=(",", ":"),
485+
)
486+
+ "\n"
487+
)
488+
self._jsonl_buffer.clear()
489+
except Exception as e:
490+
self._log_internal(f"Failed to write to JSONL: {e}", "ERROR", "red")
491+
435492
def _flush_file_buffer(self):
436493
"""Flush file buffer to disk."""
437494
if not self._file_buffer:
@@ -675,6 +732,8 @@ def flush(self):
675732
self._flush_file_buffer()
676733
if self._json_buffer:
677734
self._flush_json_buffer()
735+
if self._jsonl_buffer:
736+
self._flush_jsonl_buffer()
678737

679738
def __del__(self):
680739
"""Cleanup when logger is destroyed."""

0 commit comments

Comments
 (0)