"""High-throughput bulk stream pipe (StrfryPipe): JSONL export/import compatible with strfry.
- Streaming JSONL export (`NodeStore` -> strfry export format).
- Streaming JSONL import (`strfry export` -> `NodeStore`) with deduplication and origin tracking.
Moved verbatim from meshnode's fleetmesh_node/strfry_bridge.py; only the import paths changed.
"""
from __future__ import annotations
import json
import time
from typing import Any, Callable, Optional, TextIO
from fleetmesh_core.storage import store # noqa: E402
[docs]class StrfryPipe:
"""High-throughput bulk streaming export and import compatible with strfry."""
[docs] @staticmethod
def export_jsonl(store_backend: Any, out_stream: TextIO) -> int:
"""Stream events from NodeStore out to JSONL (strfry export format)."""
count = 0
if hasattr(store_backend, 'db') and hasattr(store_backend.db, 'execute'):
cur = store_backend.db.execute('SELECT json FROM events ORDER BY created_at ASC, id ASC')
for row in cur:
out_stream.write(row[0] + '\n')
count += 1
elif hasattr(store_backend, 'all'):
for ev in store_backend.all():
out_stream.write(json.dumps(ev, separators=(',', ':')) + '\n')
count += 1
out_stream.flush()
return count
[docs] @staticmethod
def import_jsonl(
store_backend: Any,
in_stream: TextIO,
verify: bool = True,
clock_fn: Optional[Callable[[], int]] = None,
) -> tuple[int, int, int]:
"""Stream events from JSONL (strfry export format) into NodeStore.
Returns (imported, duplicates, errors).
"""
now = (clock_fn or (lambda: int(time.time())))()
imported = 0
duplicates = 0
errors = 0
for line in in_stream:
line = line.strip()
if not line:
continue
try:
ev = json.loads(line)
if not isinstance(ev, dict) or 'id' not in ev:
errors += 1
continue
if verify and not store.verify_event(ev):
errors += 1
continue
res = store_backend.insert_event(ev, now)
if res == 'stored':
imported += 1
if hasattr(store_backend, 'note_origin'):
store_backend.note_origin(ev['id'], getattr(store_backend, 'LOCAL', 'local'))
elif res == 'duplicate':
duplicates += 1
else:
errors += 1
except Exception:
errors += 1
return imported, duplicates, errors