Source code for fleetmesh_strfry.pipe

"""High-throughput bulk stream pipe (StrfryPipe): JSONL export/import compatible with strfry.

- Streaming JSONL export (`NodeStore` -> strfry export format).
- Streaming JSONL import (`strfry export` -> `NodeStore`) with deduplication and origin tracking.

Moved verbatim from meshnode's fleetmesh_node/strfry_bridge.py; only the import paths changed.
"""
from __future__ import annotations

import json
import time
from typing import Any, Callable, Optional, TextIO
from fleetmesh_core.storage import store  # noqa: E402


[docs]class StrfryPipe: """High-throughput bulk streaming export and import compatible with strfry."""
[docs] @staticmethod def export_jsonl(store_backend: Any, out_stream: TextIO) -> int: """Stream events from NodeStore out to JSONL (strfry export format).""" count = 0 if hasattr(store_backend, 'db') and hasattr(store_backend.db, 'execute'): cur = store_backend.db.execute('SELECT json FROM events ORDER BY created_at ASC, id ASC') for row in cur: out_stream.write(row[0] + '\n') count += 1 elif hasattr(store_backend, 'all'): for ev in store_backend.all(): out_stream.write(json.dumps(ev, separators=(',', ':')) + '\n') count += 1 out_stream.flush() return count
[docs] @staticmethod def import_jsonl( store_backend: Any, in_stream: TextIO, verify: bool = True, clock_fn: Optional[Callable[[], int]] = None, ) -> tuple[int, int, int]: """Stream events from JSONL (strfry export format) into NodeStore. Returns (imported, duplicates, errors). """ now = (clock_fn or (lambda: int(time.time())))() imported = 0 duplicates = 0 errors = 0 for line in in_stream: line = line.strip() if not line: continue try: ev = json.loads(line) if not isinstance(ev, dict) or 'id' not in ev: errors += 1 continue if verify and not store.verify_event(ev): errors += 1 continue res = store_backend.insert_event(ev, now) if res == 'stored': imported += 1 if hasattr(store_backend, 'note_origin'): store_backend.note_origin(ev['id'], getattr(store_backend, 'LOCAL', 'local')) elif res == 'duplicate': duplicates += 1 else: errors += 1 except Exception: errors += 1 return imported, duplicates, errors