"""The node's event store: the reference SqliteStore, safe to share across threads.
The relay face serves every connection on its own thread and the sync
sessions run on others, all against one store. sqlite3 connections refuse
use from a thread other than the one that opened them unless told otherwise,
and even then two threads must not interleave statements. So: one
connection opened for any thread, and one lock around every primitive the
reference defines. The reference's own semantics - what supersedes what,
tombstones, ranges, fingerprints - are inherited untouched.
"""
from __future__ import annotations
from typing import Any, Callable, Iterable, Optional, TypedDict
from fleetmesh_node.compartment.schema import Column, Index, Table
from fleetmesh_node.interfaces import NostrEvent
from fleetmesh_node.storage import SqliteStore
from fleetmesh_core.storage import store
[docs]class BridgeCursorRecord(TypedDict):
relay: str
last_reconciled_at: int
last_event_at: int
events_reconciled: int
[docs]class NodeStore(SqliteStore, store.SqliteStore):
LOCAL: str = 'local'
MESH: str = 'mesh'
# The reference's own tables first, verbatim, then this node's additions.
# The connection, PRAGMAs and lock come from BaseSqliteStore; the
# reference's query semantics - what supersedes what, tombstones, ranges,
# fingerprints - come from store.SqliteStore, whose __init__ is bypassed
# because the base already opened the connection and built the schema.
SCHEMA = (
Table('events', (
Column('id', 'TEXT', primary_key=True),
Column('pubkey', 'TEXT'),
Column('created_at', 'INTEGER'),
Column('kind', 'INTEGER'),
Column('json', 'TEXT'),
), indices=(
Index('events_created_at', ('created_at', 'id')),
# Supersession and deletion ask for one author's events, and
# asked it of the whole table until this existed.
Index('events_author_kind', ('pubkey', 'kind')),
Index('events_kind_created_at', ('kind', 'created_at', 'id')),
)),
Table('tombstones', (
Column('key', 'TEXT', not_null=True),
Column('deleter', 'TEXT', not_null=True),
), primary_key=('key', 'deleter')),
# Where each event came from. Not a protocol field and not in the
# reference: the specification only says what a node may *claim*,
# and forbids asserting completeness over a shard it holds only
# through mesh replication, which is how third-party replication
# would launder unverified peer data into an authoritative record.
# Knowing which is which is this implementation's own problem.
Table('origin', (
Column('id', 'TEXT', primary_key=True),
Column('origin', 'TEXT', not_null=True),
)),
# Single-letter tags, which is the set NIP-01 indexes. A client
# asking for its own gift wraps decoded every wrap in the store
# without this.
Table('tags', (
Column('id', 'TEXT', not_null=True),
Column('name', 'TEXT', not_null=True),
Column('value', 'TEXT', not_null=True),
), primary_key=('id', 'name', 'value'), indices=(
Index('tags_name_value', ('name', 'value')),
), without_rowid=True),
# Persistent checkpoints for external relay bridge reconciliation.
Table('bridge_cursor', (
Column('relay', 'TEXT', primary_key=True),
Column('last_reconciled_at', 'INTEGER'),
Column('last_event_at', 'INTEGER'),
Column('events_reconciled', 'INTEGER', default=0),
)),
)
[docs] def __init__(self, path: str = ':memory:', **kw: Any) -> None:
SqliteStore.__init__(self, path, **kw)
# After the lock exists and not before it. _index_tags goes through
# _locked like every other writer, so backfilling above this line
# raised AttributeError on the way up and took every node down at once
# - which is the same mistake, in the same shape, as the Reporter that
# read its account store four lines above the ledger it filled.
self._backfill_tags()
self.origin_swept: int = self._sweep_origin()
def _sweep_origin(self) -> int:
"""Drop provenance rows for events this store no longer holds.
`_drop` used to leave them: every superseded replaceable event and
every NIP-09 deletion left its `origin` row behind - measured at two
events held against seven rows - and nothing ever read or removed
them (audit 2026-10-15, B3). `_drop` clears its own row now, which
stops that source. One remains by design: sync records a delivered
event's origin before deciding whether to keep it (`sync/initiator`),
so a delivery that is refused leaves a row too. Hence on every open,
not once: after the first it removes only those, in one indexed
anti-join, before anything else can write.
"""
with self._lock:
cur = self.execute('DELETE FROM origin WHERE id NOT IN (SELECT id FROM events)')
self.db.commit()
return max(0, int(cur.rowcount or 0))
@staticmethod
def _locked(name: str) -> Callable[..., Any]:
def method(self: Any, *a: Any, **k: Any) -> Any:
with self._lock:
return getattr(store.SqliteStore, name)(self, *a, **k)
method.__name__ = name
return method
_put = _locked('_put')
def _drop(self, event_id: str) -> None:
"""The reference's drop, plus this node's own provenance row.
Called by supersession and by a NIP-09 deletion, inside the one
transaction `insert_event` holds (`_atomic` joins it), so the event,
its tags and its origin go together or not at all. `forget` already
removed all three; this is the other way an event leaves (B3).
"""
with self._lock, self._atomic():
store.SqliteStore._drop(self, event_id)
self.execute('DELETE FROM origin WHERE id = ?', (event_id,))
def _all(self, q: Optional[str] = None, args: Any = ()) -> list[Any]:
if q is not None:
return SqliteStore._all(self, q, args)
with self._lock:
return store.SqliteStore._all(self)
_get = _locked('_get')
# The narrowing reads go through the lock like every other primitive: the
# relay face serves each connection on its own thread and the sync sessions
# run on others, and two threads must not interleave statements on one
# connection whatever the statement is.
_narrow = _locked('_narrow')
# Held across the read rather than around a fetchall: the point of it is
# that it stops early, and a fetchall would put the whole match set back
# in memory as the rows it was written to avoid parsing.
newest_matching = _locked('newest_matching')
_index_tags = _locked('_index_tags')
_by_author_kind = _locked('_by_author_kind')
_by_author = _locked('_by_author')
_tombstone = _locked('_tombstone')
_tombstoned_by = _locked('_tombstoned_by')
# insert_event reads then writes; hold the lock across the whole decision.
insert_event = _locked('insert_event')
[docs] def by_kind_before(self, kind: int, cutoff: int, cap: Optional[int] = None,
after: Optional[tuple[int, str]] = None) -> list[NostrEvent]:
"""Events of one kind dated before a cutoff, oldest first, one page.
For retention, which is the only caller. The index is on
(created_at, id), so this reads a range rather than the table.
`cap` and `after` are what keep the sweep from being the thing that
kills the node: a policy set for the first time on a face holding days
of gift wraps has every one of them past the cutoff at once, and this
used to answer with all of them parsed. `after` is the (created_at, id)
of the last row of the previous page, and the cursor rather than an
offset because the sweep is deleting as it walks - and because an event
held back inside a claimed shard stays, so a page that re-read what it
had refused would never end.
"""
import json as _json
where: list[str] = ['kind = ?', 'created_at < ?']
args: list[int | str] = [int(kind), int(cutoff)]
if after is not None:
where.append('(created_at, id) > (?, ?)')
args.extend([int(after[0]), str(after[1])])
sql = ('SELECT json FROM events WHERE ' + ' AND '.join(where)
+ ' ORDER BY created_at, id')
if cap is not None:
sql += ' LIMIT ?'
args.append(int(cap))
with self._lock:
rows = self.execute(sql, args).fetchall()
return [_json.loads(r[0]) for r in rows]
[docs] def count(self) -> int:
"""How many events this node holds, as one number from the index.
Written into status.json on every tick, and until this existed it was
`len(self._all())`: the whole store parsed into dicts to be thrown
away, so the node's peak heap was the size of its store and the
allocator kept it. On the public face that was 600 MB of a 1 GB box
for a number that a COUNT answers without reading a row.
"""
with self._lock:
return int(self.execute('SELECT COUNT(*) FROM events').fetchone()[0])
[docs] def forget(self, event_id: str) -> bool:
"""Drop one event because this node stopped keeping it. NOT a deletion.
A NIP-09 kind 5 tombstones an id so a later re-offer cannot resurrect
it, because there the author said the thing should not exist. Retention
says only that THIS node stopped holding it, which is a fact about a
disk and not about the event: a peer may hand it back tomorrow and be
right to. So the row goes and no tombstone is written.
"""
with self._lock:
cur = self.execute('DELETE FROM events WHERE id = ?', (event_id,))
self.execute('DELETE FROM origin WHERE id = ?', (event_id,))
self.execute('DELETE FROM tags WHERE id = ?', (event_id,))
self.db.commit()
return cur.rowcount > 0
[docs] def note_origin(self, event_id: str, origin: str) -> None:
"""Record how one event reached this node. First writer wins: an event
this node already held locally is not reclassified because a peer later
sent it too."""
if origin not in (self.LOCAL, self.MESH):
raise ValueError('origin is local or mesh')
with self._lock:
self.execute('INSERT OR IGNORE INTO origin VALUES (?,?)', (event_id, origin))
self.db.commit()
[docs] def origin_of(self, event_id: str) -> Optional[str]:
with self._lock:
row = self.execute('SELECT origin FROM origin WHERE id = ?', (event_id,)).fetchone()
return row[0] if row else None
[docs] def any_replicated(self, event_ids: Iterable[str]) -> bool:
"""True if any of these arrived through the mesh, or is of unknown origin.
Unknown counts as replicated. A claim of completeness is a falsifiable
assertion about data this node stands behind, and an event it cannot
account for the arrival of is not data it can stand behind.
"""
ids = list(event_ids)
if not ids:
return False
with self._lock:
known: dict[str, str] = {}
# Chunked: a have/need may carry validate.MAX_IDS ids, past what
# SQLite binds in one statement.
for chunk in store.chunked(ids):
sql, vals = store.in_clause('id', chunk)
known.update((r[0], r[1]) for r in self.execute(
'SELECT id, origin FROM origin WHERE ' + sql, vals))
return any(known.get(i) != self.LOCAL for i in ids)
[docs] def get_bridge_cursor(self, relay_url: str) -> Optional[BridgeCursorRecord]:
with self._lock:
cur = self.db.cursor()
cur.execute('SELECT last_reconciled_at, last_event_at, events_reconciled FROM bridge_cursor WHERE relay = ?', (relay_url,))
row = cur.fetchone()
if row is None:
return None
return {
'relay': relay_url,
'last_reconciled_at': row[0],
'last_event_at': row[1],
'events_reconciled': row[2],
}
[docs] def set_bridge_cursor(
self,
relay_url: str,
last_reconciled_at: Optional[int] = None,
last_event_at: Optional[int] = None,
count_increment: int = 0,
) -> None:
with self._lock:
cur = self.db.cursor()
cur.execute('SELECT last_reconciled_at, last_event_at, events_reconciled FROM bridge_cursor WHERE relay = ?', (relay_url,))
row = cur.fetchone()
if row is None:
cur.execute(
'INSERT INTO bridge_cursor (relay, last_reconciled_at, last_event_at, events_reconciled) VALUES (?, ?, ?, ?)',
(relay_url, last_reconciled_at or 0, last_event_at or 0, count_increment)
)
else:
new_reconciled = last_reconciled_at if last_reconciled_at is not None else row[0]
new_event_at = max(row[1] or 0, last_event_at or 0)
new_count = (row[2] or 0) + count_increment
cur.execute(
'UPDATE bridge_cursor SET last_reconciled_at = ?, last_event_at = ?, events_reconciled = ? WHERE relay = ?',
(new_reconciled, new_event_at, new_count, relay_url)
)
self.db.commit()
[docs] def all_bridge_cursors(self) -> list[BridgeCursorRecord]:
with self._lock:
cur = self.db.cursor()
cur.execute('SELECT relay, last_reconciled_at, last_event_at, events_reconciled FROM bridge_cursor ORDER BY relay')
rows = cur.fetchall()
return [
{
'relay': r[0],
'last_reconciled_at': r[1],
'last_event_at': r[2],
'events_reconciled': r[3],
}
for r in rows
]