Source code for fleetmesh_core.storage.store

"""MeshStore: what a storage backend must agree with every other one about.

The specification requires that every backend produce byte-identical range
fingerprints for identical event sets, because two nodes whose fingerprints
disagree cannot reconcile and cannot tell that from having different data.

The fingerprint algorithm is NIP-77's and is pinned in fleetmesh_core/storage/replication.py and
vectors/fingerprint.json. That is only half of what agreement needs. The other
half is *which events are in the set*, and writing this module is what found
that the prose never said. Four rules decide it, all now in constants.json
under `store`, and all four are places where two honest implementations would
otherwise diverge:

  1. ephemeral kinds are never stored, so they never enter a fingerprint
  2. replaceable and addressable kinds keep only the winner
  3. a valid NIP-09 deletion removes its target, permanently - and a deletion
     that arrives before its target still binds, which is why a tombstone
     records *who* asked rather than merely that someone did: authority
     depends on the target's author, and that is not known yet
  4. an expired event is not in the set

Standard library only. Two backends, one behavioural suite: core/tests/test_store.py
runs the same assertions against both, which is the store conformance suite the
specification's Conformance section asks for.
"""
from __future__ import annotations

import contextlib
import hashlib
import json
import sqlite3
from typing import Any, ContextManager, Iterator, Mapping, Optional, Sequence, Union

from . import replication

from ..crypto import crypto

from ..wire import schema_files
from ..wire.interfaces import NostrEvent, TagList, UnsignedNostrEvent
from ..wire import spelling

# Above this many values an IN list is bound as one JSON array and read
# back with json_each, so the statement costs one bound variable whatever
# the list's length. Below it the plain `IN (?,?,...)` form is kept: it is
# what the planner has statistics for and what every existing test reads.
# The limit this protects is SQLITE_LIMIT_VARIABLE_NUMBER (999 on old
# builds, 32766 on current ones), while validate.MAX_IDS lets a peer send
# 20000 ids in one have/need and a REQ carries whatever the relay's
# `authors` list holds. A list past the limit was an OperationalError the
# peer could cause on demand.
IN_LIST_INLINE_MAX: int = 500


[docs]def in_clause(column: str, values: Sequence[Any]) -> tuple[str, list[Any]]: """`(sql, args)` for `column IN <values>` with a bounded variable count.""" vs = list(values) if len(vs) <= IN_LIST_INLINE_MAX: return '%s IN (%s)' % (column, ','.join('?' * len(vs))), vs return '%s IN (SELECT value FROM json_each(?))' % column, [json.dumps(vs)]
[docs]def chunked(values: Sequence[Any], size: int = IN_LIST_INLINE_MAX): """Yield `values` in runs of at most `size`, for statements run per chunk.""" vs = list(values) for i in range(0, len(vs), size): yield vs[i:i + size]
CONSTANTS = schema_files.load_constants() STORE = CONSTANTS['store'] # The module cannot silently disagree with the schema the protocol version is a # digest over: if a rule below is reworded away from what this code does, this # fails at import rather than at the first divergent fingerprint. assert STORE['fingerprint_bytes'] == 16 assert 'never stored' in STORE['ephemeral_not_stored'] assert 'lexicographically lower' in STORE['supersession'] assert 'intersection' in STORE['range'] assert 'created_at ascending' in STORE['enumeration_order'] # ------------------------------------------------------------- NIP-01 kinds --
[docs]def is_ephemeral(kind: int) -> bool: return 20000 <= kind < 30000
[docs]def is_replaceable(kind: int) -> bool: return kind in (0, 3) or 10000 <= kind < 20000
[docs]def is_addressable(kind: int) -> bool: return 30000 <= kind < 40000
[docs]def ascii_int(value: Any) -> Optional[int]: """The integer a wire string spells, or None if it is not one. A wire integer is one or more ASCII digits and nothing else: no sign, no whitespace, no underscores, no other script's digits. Python's int() and str.isdigit() accept all of those, and a second implementation's parser accepts a different subset, so a value one side read as a number the other read as garbage. This is the one spelling both agree on. """ if not isinstance(value, str) or not value: return None if not all('0' <= c <= '9' for c in value): return None return int(value)
[docs]def is_hex64(value: Any) -> bool: """Exactly 64 lowercase hex digits: the spelling of an id or a pubkey.""" return spelling.is_hex(value, 64)
[docs]def is_int(value: Any) -> bool: """A JSON integer: an int, and not a bool (which is one to Python).""" return spelling.is_int(value)
[docs]def tags(event: Mapping[str, Any], name: str) -> list[list[str]]: """Every tag of one name, in order. Three modules had written this for themselves before it was worth having once. A tag whose elements are not all strings is skipped: NIP-01 tags are arrays of strings, and a reader that indexed into one of another shape crashed instead of reporting.""" raw = event.get('tags') if isinstance(event, Mapping) else None if not isinstance(raw, list): return [] return [list(t) for t in raw if isinstance(t, list) and t and t[0] == name and all(isinstance(x, str) for x in t)]
[docs]def first_tag(event: Mapping[str, Any], name: str) -> Optional[list[str]]: """The first tag of one name, or None. A reader wanting the value takes `t[1]` and checks the length first: a tag with no value is well-formed NIP-01 and means nothing here.""" found = tags(event, name) return found[0] if found else None
[docs]def d_tag(event: Mapping[str, Any]) -> str: for t in event.get('tags', []): if t and t[0] == 'd': return t[1] if len(t) > 1 else '' return ''
[docs]def coordinate(event: Mapping[str, Any]) -> Optional[str]: """The key supersession is resolved within, or None for a regular event. This is the NIP-01 coordinate, `kind:pubkey:d` - the same string a NIP-09 `a` tag carries - so a deletion by coordinate can be matched against it byte for byte. A replaceable kind has an empty d: `kind:pubkey:`.""" k = event['kind'] if is_replaceable(k): return '%d:%s:' % (k, event['pubkey']) if is_addressable(k): return '%d:%s:%s' % (k, event['pubkey'], d_tag(event)) return None
def _coordinates(deletion: Mapping[str, Any]) -> set[str]: """The `a` tags of a kind 5 whose pubkey is the deleter's own. The coordinate's middle field is compared as a string: a coordinate is a name, and `kind:PUBKEY:d` in another case is not the same name.""" out = set() for t in deletion.get('tags', []): if len(t) > 1 and t[0] == 'a' and isinstance(t[1], str): parts = t[1].split(':', 2) if len(parts) == 3 and parts[1] == deletion['pubkey']: out.add(t[1]) return out
[docs]def expiration(event: Mapping[str, Any]) -> Optional[int]: """NIP-40, or None. The first well-formed expiration tag wins; a value that is not ASCII digits is not an expiration tag at all, so it is skipped rather than read as expiry now: a peer cannot be allowed to evict an event by sending garbage, and two implementations with different int parsers cannot be allowed to disagree about whether it was garbage.""" for t in tags(event, 'expiration'): if len(t) > 1: ts = ascii_int(t[1]) if ts is not None: return ts return None
[docs]def supersedes(new: Mapping[str, Any], old: Mapping[str, Any]) -> bool: """NIP-01: later created_at wins; on a tie the lexicographically lower id. The tie-break matters more than it looks. Two nodes that saw both versions must pick the same one or their fingerprints differ forever, and a tie is exactly what a replaceable event republished within one second produces. """ if new['created_at'] != old['created_at']: return new['created_at'] > old['created_at'] return new['id'] < old['id']
# ------------------------------------------------------------ NIP-01 filter --
[docs]def matches(event: Mapping[str, Any], filt: Mapping[str, Any]) -> bool: """NIP-01 filter matching, including single-letter #tag filters. Every condition is an AND across fields and an OR within a field, which is NIP-01's rule and the one thing a backend cannot get creative about. """ if 'ids' in filt and event['id'] not in filt['ids']: return False if 'authors' in filt and event['pubkey'] not in filt['authors']: return False if 'kinds' in filt and event['kind'] not in filt['kinds']: return False if 'since' in filt and event['created_at'] < filt['since']: return False if 'until' in filt and event['created_at'] > filt['until']: return False for key, wanted in filt.items(): if not key.startswith('#') or len(key) != 2: continue name = key[1] values = {t[1] for t in event.get('tags', []) if len(t) > 1 and t[0] == name} if values.isdisjoint(wanted): return False return True
_SCALARS = (str, int, float, bool, type(None)) # What a compiled filter holds under a key: a set of ids, authors, kinds or # tag values; a list left as it came (one holding something unhashable); or # a scalar, such as a search term. FilterValue = Union[frozenset[Union[str, int, float, bool, None]], Sequence[Any], int, str, None]
[docs]def compiled(filt: Mapping[str, Any]) -> Mapping[str, FilterValue]: """`filt` without since/until/limit, its lists as sets: the form `matches` answers fastest when it is asked about many events. `_in_range` and `newest_matching` built that reduced filter again for every event, and `matches` tested `ids` and `authors` as lists - O(n) per event, with up to validate.MAX_IDS ids - and rebuilt each tag set per event per tag (audit 2026-10-15, E2). Same answers: membership is membership. A list holding anything but scalars is left a list, and answered as before. """ out: dict[str, FilterValue] = {} for k, v in filt.items(): if k in ('since', 'until', 'limit'): continue setlike = (k in ('ids', 'authors', 'kinds') or k.startswith('#')) and isinstance(v, (list, tuple)) out[k] = frozenset(v) if setlike and all(isinstance(x, _SCALARS) for x in v) else v return out
[docs]class MeshStore: """The interface, and every rule that is the same whatever the backend. A backend supplies raw storage: _put, _drop, _all, _get, _tombstone, _is_tombstoned. Everything that decides set membership lives here, so a new backend cannot accidentally invent its own answer to the four questions above - which is the whole point of a shared suite. """ # ---- backend surface ------------------------------------------------- def _put(self, event: Mapping[str, Any]) -> None: raise NotImplementedError def _drop(self, event_id: str) -> None: raise NotImplementedError def _all(self) -> list[Mapping[str, Any]]: raise NotImplementedError def _get(self, event_id: str) -> Optional[Mapping[str, Any]]: raise NotImplementedError # Narrowing is optional: a backend that cannot do better than reading # everything inherits these and is still correct. A backend that can - one # with an index - overrides them, and the shared suite proves the two give # the same answers. # # This is not an optimisation for its own sake. Every scan below was # `self._all()`, which materialises the whole store as Python dicts, so a # node paid O(store) on every query AND on every replaceable insert. On a # node with a public face taking about 150 events an hour it cost roughly # 50 MB of RSS an hour against an 8,000-event store, because CPython does # not return freed arenas to the system, and only a restart brought it # back. A relay that degrades quadratically in the size of its own shard is # a relay that cannot hold a large one. def _narrow(self, lo: int, hi: int, kinds: Optional[Sequence[int]] = None, authors: Optional[Sequence[str]] = None, tags: Optional[Mapping[str, Sequence[str]]] = None, ids: Optional[Sequence[str]] = None) -> list[Mapping[str, Any]]: """Everything in [lo, hi], optionally of these kinds, authors, tags or ids. Narrowing may return a superset and must never return less: `matches` above decides membership, and this only says where to look. A backend that narrows none of it is still correct, which is what these defaults are for. """ out = [e for e in self._all() if lo <= e['created_at'] <= hi] if kinds is not None: ks = set(kinds) out = [e for e in out if e['kind'] in ks] if authors is not None: aus = set(authors) out = [e for e in out if e['pubkey'] in aus] if ids is not None: want_ids = set(ids) out = [e for e in out if e['id'] in want_ids] for name, values in (tags or {}).items(): want = set(values) out = [e for e in out if any(len(t) > 1 and t[0] == name and t[1] in want for t in e.get('tags', []))] return out def _by_author_kind(self, pubkey: str, kind: int) -> list[Mapping[str, Any]]: """Every event one author wrote of one kind. Order is not promised.""" return [e for e in self._all() if e['pubkey'] == pubkey and e['kind'] == kind] def _by_author(self, pubkey: str) -> list[Mapping[str, Any]]: """Every event one author wrote. Order is not promised.""" return [e for e in self._all() if e['pubkey'] == pubkey] def _tombstone(self, key: str, deleter: str) -> None: raise NotImplementedError def _tombstoned_by(self, key: str) -> set[str]: raise NotImplementedError def _atomic(self) -> ContextManager[None]: """One transaction around a decision that writes more than once. Supersession drops the old version and puts the new one; a deletion drops each target, tombstones it, and stores itself. Each primitive used to commit on its own, so a process that died between two of them left a coordinate with no winner, or a dropped event with no tombstone - which the next peer to offer it would resurrect (audit 2026-10-15, B4). A backend with transactions overrides this so `insert_event` is all or nothing. The default is for a backend with nothing to roll back. """ return contextlib.nullcontext() # ---- interface -------------------------------------------------------
[docs] def get_event(self, event_id: str) -> Optional[Mapping[str, Any]]: return self._get(event_id)
[docs] def has_event(self, event_id: str) -> bool: return self._get(event_id) is not None
[docs] def insert_event(self, event: Mapping[str, Any], now: int) -> str: """Returns one of: stored, duplicate, ephemeral, expired, superseded, deleted, or a deletion outcome. `now` is explicit rather than read from the clock so that two backends fed the same history agree exactly. All or nothing, through `_atomic`: every write one insertion makes lands together or not at all. """ with self._atomic(): return self._insert_event(event, now)
def _insert_event(self, event: Mapping[str, Any], now: int) -> str: eid = event['id'] # A deletion may have arrived first. Its authority could not be checked # then, because that needs this event's author; it is checked now. # A kind 5 is never deleted, so a tombstone naming one is ignored # in either arrival order. if event['kind'] != 5 and event['pubkey'] in self._tombstoned_by(eid): return 'deleted' coord = coordinate(event) if coord is not None and self._deleted_by_coordinate(event, coord): return 'deleted' if self._get(eid) is not None: return 'duplicate' if is_ephemeral(event['kind']): return 'ephemeral' exp = expiration(event) if exp is not None and exp <= now: return 'expired' if event['kind'] == 5: return self._apply_deletion(event) if coord is not None: # A coordinate is (kind, pubkey, d), so the author and the kind # narrow it to a handful before anything is decoded and compared. for other in list(self._by_author_kind(event['pubkey'], event['kind'])): if coordinate(other) == coord: if supersedes(event, other): self._drop(other['id']) else: return 'superseded' self._put(event) return 'stored' def _deleted_by_coordinate(self, event: Mapping[str, Any], coord: str) -> bool: """NIP-09 by coordinate, checked on arrival: a kind 5 the same author signed, naming this coordinate in an `a` tag, and created no earlier than the event. The kind 5 is stored, so it is the record; a coordinate is a name for a *series* and a later version of the series is new content the deletion did not speak about.""" return any(coord in _coordinates(d) and event['created_at'] <= d['created_at'] for d in self._by_author_kind(event['pubkey'], 5)) def _apply_deletion(self, deletion: Mapping[str, Any]) -> str: """NIP-09, with the authority check the specification makes normative: a deletion may only remove events its own author signed. The kind 5 is itself stored, because a peer that never receives it never learns of the deletion, and tombstones are kept so a later re-offer of the same id cannot resurrect it. An `a` tag is a NIP-01 coordinate, `kind:pubkey:d`, and reaches only versions created at or before the deletion. An `a` tag whose pubkey is not the deleter's names nothing it has authority over and is ignored. A kind 5 cannot itself be deleted: a deletion of a deletion has no effect, or a peer could be made to forget what it was told to forget.""" removed = 0 targets = {t[1] for t in deletion.get('tags', []) if len(t) > 1 and t[0] == 'e'} coords = _coordinates(deletion) # Only the deleter's own events can be affected - the authority check # below is normative, not an optimisation - so that is the set to read. for other in list(self._by_author(deletion['pubkey'])): if other['kind'] == 5: continue # a deletion is never deleted hit = other['id'] in targets or (coordinate(other) in coords and other['created_at'] <= deletion['created_at']) if not hit: continue if other['pubkey'] != deletion['pubkey']: continue # a stranger's deletion is not authority over this event self._drop(other['id']) self._tombstone(other['id'], deletion['pubkey']) removed += 1 # Targets not held yet. Recording the deleter's key is what lets the # authority check happen on arrival instead of being skipped: a # deletion that names a stranger's event tombstones nothing it can # enforce, whichever order the two events turn up in. Coordinates need # no tombstone: the stored kind 5 is consulted on arrival. for t in targets: if self._get(t) is None: self._tombstone(t, deletion['pubkey']) self._put(deletion) return 'deleted %d' % removed # ---- range queries --------------------------------------------------- def _in_range(self, filt: Mapping[str, Any], since: int, until: int, now: int) -> list[Mapping[str, Any]]: """The set. `since`/`until` bound the time axis and a filter's own since/until narrows further - the intersection, never the wider of the two, so that neither argument can silently widen the other.""" lo, hi = since, until if 'since' in filt: lo = max(lo, filt['since']) if 'until' in filt: hi = min(hi, filt['until']) out: list[Mapping[str, Any]] = [] # Narrowed by the backend where it can be, then decided here. The time # bound is the one narrowing every caller supplies, and it is the one # the events table is indexed on. # NIP-01 indexes single-letter tags and only those, which is exactly # the set a backend can be asked to narrow on. want_tags = {k[1:]: v for k, v in filt.items() if len(k) == 2 and k[0] == '#' and isinstance(v, (list, tuple))} rest = compiled(filt) for ev in self._narrow(lo, hi, filt.get('kinds'), filt.get('authors'), want_tags, filt.get('ids')): exp = expiration(ev) if exp is not None and exp <= now: continue if not matches(ev, rest): continue out.append(ev) return out
[docs] def enumerate_range(self, filt: Mapping[str, Any], since: int, until: int, now: int) -> list[str]: """(created_at ASC, id ASC). The order is observable, so it is normative even though the fingerprint does not need it.""" return [e['id'] for e in sorted(self._in_range(filt, since, until, now), key=lambda e: (e['created_at'], e['id']))]
[docs] def items_range(self, filt: Mapping[str, Any], since: int, until: int, now: int) -> list[tuple[int, str]]: """(created_at, id) pairs in the same order, which is the record shape negentropy reconciles over - timestamp first, id second, ascending.""" return [(e['created_at'], e['id']) for e in sorted(self._in_range(filt, since, until, now), key=lambda e: (e['created_at'], e['id']))]
[docs] def fingerprint_range(self, filt: Mapping[str, Any], since: int, until: int, now: int) -> str: """NIP-77's fingerprint over that set. 16 bytes, as 32 hex characters.""" return replication.fingerprint(self.enumerate_range(filt, since, until, now))
[docs] def newest_matching(self, filt: Mapping[str, Any], since: int, until: int, now: int, cap: int) -> list[Mapping[str, Any]]: """At most `cap` matching events, newest first, ties broken by id ascending. Every other range query answers "the whole set", which is the only answer a fingerprint or a reconciliation can use. A REQ is the one caller that knows up front how many it will send, and on an open face the set can be four orders of magnitude larger than the limit. Asking for the set and then slicing it is what a backend cannot afford, so the question is asked in the shape the answer is wanted in. """ ordered = sorted(self._in_range(filt, since, until, now), key=lambda e: (-e['created_at'], e['id'])) return ordered[:max(0, cap)]
[docs]class MemoryStore(MeshStore): """A dict. The naive backend the specification says is conformant."""
[docs] def __init__(self) -> None: self._events: dict[str, Mapping[str, Any]] = {} self._tombs: dict[str, set[str]] = {}
def _put(self, event: Mapping[str, Any]) -> None: self._events[event['id']] = dict(event) def _drop(self, event_id: str) -> None: self._events.pop(event_id, None) def _all(self) -> list[Mapping[str, Any]]: return list(self._events.values()) def _get(self, event_id: str) -> Optional[Mapping[str, Any]]: return self._events.get(event_id) def _tombstone(self, key: str, deleter: str) -> None: self._tombs.setdefault(key, set()).add(deleter) def _tombstoned_by(self, key: str) -> set[str]: return self._tombs.get(key, set())
[docs]class SqliteStore(MeshStore): """sqlite3 from the standard library, storing the event as canonical JSON. Deliberately naive: it scans. The specification is explicit that how a backend computes a range is unspecified and only the value on the wire is normative, so a backend that scans and one with a prefix-sum index must agree - and this one exists to be the slow, obviously-correct side of that comparison. """
[docs] def __init__(self, path: str = ':memory:') -> None: self.db = sqlite3.connect(path) self.db.executescript( 'CREATE TABLE IF NOT EXISTS events (' ' id TEXT PRIMARY KEY, pubkey TEXT, created_at INTEGER, kind INTEGER, json TEXT);' 'CREATE INDEX IF NOT EXISTS events_created_at ON events (created_at, id);' # Supersession asks for one author's events of one kind on every # replaceable insert, and a deletion asks for one author's events. # Without this both are a table scan, which is what they were. 'CREATE INDEX IF NOT EXISTS events_author_kind ON events (pubkey, kind);' 'CREATE TABLE IF NOT EXISTS tombstones (' ' key TEXT, deleter TEXT, PRIMARY KEY (key, deleter));' # NIP-01 indexes single-letter tags and only those, so that is what # is kept here. Without it a client asking for its own gift wraps - # kinds 1059, #p its key - decoded every wrap in the store to find # none, which on a public face was half a second and eight # megabytes per subscription, repeated per client. 'CREATE TABLE IF NOT EXISTS tags (' ' id TEXT, name TEXT, value TEXT, PRIMARY KEY (id, name, value)) WITHOUT ROWID;' 'CREATE INDEX IF NOT EXISTS tags_name_value ON tags (name, value);') self.db.commit() self._backfill_tags()
def _backfill_tags(self) -> None: """Index the tags of events stored before this table existed. A store opened by an older build has events and no rows here, and an empty index is indistinguishable from an index over events with no tags - so the question asked is whether any event is missing from it, not whether the table is empty. """ missing = self.db.execute( 'SELECT json FROM events WHERE id NOT IN (SELECT id FROM tags)').fetchall() if not missing: return for (raw,) in missing: self._index_tags(json.loads(raw)) self.db.commit() def _index_tags(self, event: Mapping[str, Any]) -> None: rows = {(event['id'], t[0], t[1]) for t in event.get('tags', []) if len(t) > 1 and len(t[0]) == 1 and isinstance(t[1], str)} # An event with no single-letter tag still gets one row, with a null # name, or _backfill_tags would rediscover it as missing on every open. self.db.executemany('INSERT OR REPLACE INTO tags VALUES (?,?,?)', rows or [(event['id'], '', '')]) # Nesting depth of `_atomic`. A class attribute so a subclass that opens # the connection itself, and skips this __init__, starts at zero too. _atomic_depth: int = 0 def _commit(self) -> None: """Commit, unless an `_atomic` block will commit for us.""" if self._atomic_depth == 0: self.db.commit() @contextlib.contextmanager def _atomic(self) -> Iterator[None]: """BEGIN on the outermost block; COMMIT if it returns, ROLLBACK if it raises. Inner blocks join the outer one. BEGIN is explicit so this holds whatever isolation mode the connection was opened in.""" outer = self._atomic_depth == 0 if outer and not self.db.in_transaction: self.db.execute('BEGIN') self._atomic_depth += 1 try: yield except BaseException: self._atomic_depth -= 1 if outer: self.db.rollback() raise self._atomic_depth -= 1 if outer: self.db.commit() def _put(self, event: Mapping[str, Any]) -> None: self.db.execute('INSERT OR REPLACE INTO events VALUES (?,?,?,?,?)', (event['id'], event['pubkey'], event['created_at'], event['kind'], crypto.canonical(event).decode())) self._index_tags(event) self._commit() def _drop(self, event_id: str) -> None: self.db.execute('DELETE FROM events WHERE id = ?', (event_id,)) self.db.execute('DELETE FROM tags WHERE id = ?', (event_id,)) self._commit() def _all(self) -> list[Mapping[str, Any]]: return [json.loads(r[0]) for r in self.db.execute('SELECT json FROM events')] def _where(self, lo: int, hi: int, kinds: Optional[Sequence[int]] = None, authors: Optional[Sequence[str]] = None, tags: Optional[Mapping[str, Sequence[str]]] = None, ids: Optional[Sequence[str]] = None ) -> Optional[tuple[str, list[Any]]]: # A coverage claim fingerprints a shard over an unbounded range, so the # time bound alone left it reading the whole store to answer with 38 # ids. kind and author are columns; the tag it also filters on is not, # and stays in `matches`. None means no row can match. where: list[str] = ['created_at BETWEEN ? AND ?'] args: list[Any] = [int(lo), int(hi)] if kinds is not None: ks = list(kinds) if not ks: return None sql, vals = in_clause('kind', [int(k) for k in ks]) where.append(sql) args.extend(vals) if authors is not None: aus = list(authors) if not aus: return None sql, vals = in_clause('pubkey', aus) where.append(sql) args.extend(vals) # A REQ by id read every event in the time range, newest first, until # one matched: 36 ms for one missing id on 20,000 events, against # 0.03 ms by primary key, under the store lock and on demand for any # client (audit 2026-10-15, E1). An id that is not a string can never # equal one, so dropping it changes no answer. if ids is not None: wanted = [i for i in ids if isinstance(i, str)] if not wanted: return None sql, vals = in_clause('id', wanted) where.append(sql) args.extend(vals) # One EXISTS per tag name, which is NIP-01's own semantics: values # within a name are OR, and different names are AND. for name, values in (tags or {}).items(): vs = list(values) if not vs: return None sql, vals = in_clause('tags.value', vs) where.append('EXISTS (SELECT 1 FROM tags WHERE tags.id = events.id ' 'AND tags.name = ? AND %s)' % sql) args.append(name) args.extend(vals) return ' AND '.join(where), args def _narrow(self, lo: int, hi: int, kinds: Optional[Sequence[int]] = None, authors: Optional[Sequence[str]] = None, tags: Optional[Mapping[str, Sequence[str]]] = None, ids: Optional[Sequence[str]] = None) -> list[Mapping[str, Any]]: clause = self._where(lo, hi, kinds, authors, tags, ids) if clause is None: return [] return [json.loads(r[0]) for r in self.db.execute( 'SELECT json FROM events WHERE ' + clause[0], clause[1])]
[docs] def newest_matching(self, filt: Mapping[str, Any], since: int, until: int, now: int, cap: int) -> list[Mapping[str, Any]]: """The bounded answer, read newest first and stopped at `cap`. The inherited version narrows in SQL and then holds every match as a parsed event so it can be sorted. That is the whole store on an open face - which is how a 1 GB box serving 8,923 events died on a REQ for the newest twenty. Here sqlite does the ordering on the index the events table already has, and rows are parsed one at a time until enough of them have matched. """ if cap <= 0: return [] lo, hi = since, until if 'since' in filt: lo = max(lo, filt['since']) if 'until' in filt: hi = min(hi, filt['until']) want_tags = {k[1:]: v for k, v in filt.items() if len(k) == 2 and k[0] == '#' and isinstance(v, (list, tuple))} clause = self._where(lo, hi, filt.get('kinds'), filt.get('authors'), want_tags, filt.get('ids')) if clause is None: return [] rest = compiled(filt) out: list[Mapping[str, Any]] = [] cursor = self.db.execute( 'SELECT json FROM events WHERE ' + clause[0] + ' ORDER BY created_at DESC, id ASC', clause[1]) for row in cursor: ev = json.loads(row[0]) exp = expiration(ev) if exp is not None and exp <= now: continue if not matches(ev, rest): continue out.append(ev) if len(out) >= cap: break return out
def _by_author_kind(self, pubkey: str, kind: int) -> list[Mapping[str, Any]]: return [json.loads(r[0]) for r in self.db.execute( 'SELECT json FROM events WHERE pubkey = ? AND kind = ?', (pubkey, int(kind)))] def _by_author(self, pubkey: str) -> list[Mapping[str, Any]]: return [json.loads(r[0]) for r in self.db.execute( 'SELECT json FROM events WHERE pubkey = ?', (pubkey,))] def _get(self, event_id: str) -> Optional[Mapping[str, Any]]: r = self.db.execute('SELECT json FROM events WHERE id = ?', (event_id,)).fetchone() return json.loads(r[0]) if r else None def _tombstone(self, key: str, deleter: str) -> None: self.db.execute('INSERT OR IGNORE INTO tombstones VALUES (?,?)', (key, deleter)) self._commit() def _tombstoned_by(self, key: str) -> set[str]: return {r[0] for r in self.db.execute( 'SELECT deleter FROM tombstones WHERE key = ?', (key,))}
[docs] def close(self) -> None: """Safely close the underlying database connection.""" if hasattr(self, 'db') and self.db is not None: try: self.db.close() except Exception: pass
def __del__(self) -> None: try: self.close() except Exception: pass
[docs]def event_id(pubkey: str, created_at: int, kind: int, tags: Optional[Sequence[Sequence[str]]] = None, content: str = '') -> str: """NIP-01 event id: sha256 over the canonical serialisation.""" tags = tags or [] ser = json.dumps([0, pubkey, created_at, kind, tags, content], separators=(',', ':'), ensure_ascii=False) return hashlib.sha256(ser.encode()).hexdigest()
[docs]def make_event(pubkey: str, created_at: int, kind: int, tags: Optional[Sequence[Sequence[str]]] = None, content: str = '') -> UnsignedNostrEvent: """A well-formed event without a signature. The store never checks one: the interface takes "an independently authenticated event", so verification happens before insertion and belongs to whatever did the authenticating. That makes this cheap enough to build corpora of thousands, which the tests need. It is *not* a complete nostr event - NIP-01 requires a sig, and anything that puts an event on the wire needs make_signed_event instead. """ tag_list: TagList = [list(t) for t in (tags or [])] return {'id': event_id(pubkey, created_at, kind, tag_list, content), 'pubkey': pubkey, 'created_at': created_at, 'kind': kind, 'tags': tag_list, 'content': content}
[docs]def make_signed_event(sk_hex: str, created_at: int, kind: int, tags: Optional[Sequence[Sequence[str]]] = None, content: str = '') -> NostrEvent: """A complete NIP-01 event, signature and all. Slow - the curve arithmetic here is pure Python - so it is for the places that need a real event rather than a corpus: anything validated against schemas/common/nostr-event.schema.json, or put in a sync/1.0/deliver. """ from ..crypto import crypto unsigned = make_event(crypto.pubkey(sk_hex), created_at, kind, tags, content) # Built key by key rather than by copying the unsigned dict: a TypedDict # with a required `sig` cannot be made from one without it, and the # checker holds us to that. return {'id': unsigned['id'], 'pubkey': unsigned['pubkey'], 'created_at': unsigned['created_at'], 'kind': unsigned['kind'], 'tags': unsigned['tags'], 'content': unsigned['content'], 'sig': crypto.schnorr_sign(bytes.fromhex(unsigned['id']), sk_hex)}
[docs]def verify_event(ev: Mapping[str, Any]) -> bool: """NIP-01: the id is the hash of the serialised event and the signature is BIP-340 over that id by the event's pubkey. False for anything malformed, which includes a missing `tags` or `content`: NIP-01 requires both, and an event without them is not one that another implementation would hash the same way.""" from ..crypto import crypto from ..wire.wire_types import is_event_shaped if not is_event_shaped(ev) or not is_hex64(ev['pubkey']) or not is_hex64(ev['id']) or ev['created_at'] < 0: return False try: want = event_id(ev['pubkey'], ev['created_at'], ev['kind'], ev['tags'], ev['content']) return ev['id'] == want and crypto.schnorr_verify(bytes.fromhex(want), ev['pubkey'], ev['sig']) except (KeyError, ValueError, TypeError, AttributeError): return False