"""MeshStore: what a storage backend must agree with every other one about.
The specification requires that every backend produce byte-identical range
fingerprints for identical event sets, because two nodes whose fingerprints
disagree cannot reconcile and cannot tell that from having different data.
The fingerprint algorithm is NIP-77's and is pinned in fleetmesh_core/storage/replication.py and
vectors/fingerprint.json. That is only half of what agreement needs. The other
half is *which events are in the set*, and writing this module is what found
that the prose never said. Four rules decide it, all now in constants.json
under `store`, and all four are places where two honest implementations would
otherwise diverge:
1. ephemeral kinds are never stored, so they never enter a fingerprint
2. replaceable and addressable kinds keep only the winner
3. a valid NIP-09 deletion removes its target, permanently - and a deletion
that arrives before its target still binds, which is why a tombstone
records *who* asked rather than merely that someone did: authority
depends on the target's author, and that is not known yet
4. an expired event is not in the set
Standard library only. Two backends, one behavioural suite: core/tests/test_store.py
runs the same assertions against both, which is the store conformance suite the
specification's Conformance section asks for.
"""
from __future__ import annotations
import contextlib
import hashlib
import json
import sqlite3
from typing import Any, ContextManager, Iterator, Mapping, Optional, Sequence, Union
from . import replication
from ..crypto import crypto
from ..wire import schema_files
from ..wire.interfaces import NostrEvent, TagList, UnsignedNostrEvent
from ..wire import spelling
# Above this many values an IN list is bound as one JSON array and read
# back with json_each, so the statement costs one bound variable whatever
# the list's length. Below it the plain `IN (?,?,...)` form is kept: it is
# what the planner has statistics for and what every existing test reads.
# The limit this protects is SQLITE_LIMIT_VARIABLE_NUMBER (999 on old
# builds, 32766 on current ones), while validate.MAX_IDS lets a peer send
# 20000 ids in one have/need and a REQ carries whatever the relay's
# `authors` list holds. A list past the limit was an OperationalError the
# peer could cause on demand.
IN_LIST_INLINE_MAX: int = 500
[docs]def in_clause(column: str, values: Sequence[Any]) -> tuple[str, list[Any]]:
"""`(sql, args)` for `column IN <values>` with a bounded variable count."""
vs = list(values)
if len(vs) <= IN_LIST_INLINE_MAX:
return '%s IN (%s)' % (column, ','.join('?' * len(vs))), vs
return '%s IN (SELECT value FROM json_each(?))' % column, [json.dumps(vs)]
[docs]def chunked(values: Sequence[Any], size: int = IN_LIST_INLINE_MAX):
"""Yield `values` in runs of at most `size`, for statements run per chunk."""
vs = list(values)
for i in range(0, len(vs), size):
yield vs[i:i + size]
CONSTANTS = schema_files.load_constants()
STORE = CONSTANTS['store']
# The module cannot silently disagree with the schema the protocol version is a
# digest over: if a rule below is reworded away from what this code does, this
# fails at import rather than at the first divergent fingerprint.
assert STORE['fingerprint_bytes'] == 16
assert 'never stored' in STORE['ephemeral_not_stored']
assert 'lexicographically lower' in STORE['supersession']
assert 'intersection' in STORE['range']
assert 'created_at ascending' in STORE['enumeration_order']
# ------------------------------------------------------------- NIP-01 kinds --
[docs]def is_ephemeral(kind: int) -> bool:
return 20000 <= kind < 30000
[docs]def is_replaceable(kind: int) -> bool:
return kind in (0, 3) or 10000 <= kind < 20000
[docs]def is_addressable(kind: int) -> bool:
return 30000 <= kind < 40000
[docs]def ascii_int(value: Any) -> Optional[int]:
"""The integer a wire string spells, or None if it is not one.
A wire integer is one or more ASCII digits and nothing else: no sign, no
whitespace, no underscores, no other script's digits. Python's int() and
str.isdigit() accept all of those, and a second implementation's parser
accepts a different subset, so a value one side read as a number the
other read as garbage. This is the one spelling both agree on.
"""
if not isinstance(value, str) or not value:
return None
if not all('0' <= c <= '9' for c in value):
return None
return int(value)
[docs]def is_hex64(value: Any) -> bool:
"""Exactly 64 lowercase hex digits: the spelling of an id or a pubkey."""
return spelling.is_hex(value, 64)
[docs]def is_int(value: Any) -> bool:
"""A JSON integer: an int, and not a bool (which is one to Python)."""
return spelling.is_int(value)
[docs]def first_tag(event: Mapping[str, Any], name: str) -> Optional[list[str]]:
"""The first tag of one name, or None. A reader wanting the value takes
`t[1]` and checks the length first: a tag with no value is well-formed
NIP-01 and means nothing here."""
found = tags(event, name)
return found[0] if found else None
[docs]def d_tag(event: Mapping[str, Any]) -> str:
for t in event.get('tags', []):
if t and t[0] == 'd':
return t[1] if len(t) > 1 else ''
return ''
[docs]def coordinate(event: Mapping[str, Any]) -> Optional[str]:
"""The key supersession is resolved within, or None for a regular event.
This is the NIP-01 coordinate, `kind:pubkey:d` - the same string a NIP-09
`a` tag carries - so a deletion by coordinate can be matched against it
byte for byte. A replaceable kind has an empty d: `kind:pubkey:`."""
k = event['kind']
if is_replaceable(k):
return '%d:%s:' % (k, event['pubkey'])
if is_addressable(k):
return '%d:%s:%s' % (k, event['pubkey'], d_tag(event))
return None
def _coordinates(deletion: Mapping[str, Any]) -> set[str]:
"""The `a` tags of a kind 5 whose pubkey is the deleter's own. The
coordinate's middle field is compared as a string: a coordinate is a
name, and `kind:PUBKEY:d` in another case is not the same name."""
out = set()
for t in deletion.get('tags', []):
if len(t) > 1 and t[0] == 'a' and isinstance(t[1], str):
parts = t[1].split(':', 2)
if len(parts) == 3 and parts[1] == deletion['pubkey']:
out.add(t[1])
return out
[docs]def expiration(event: Mapping[str, Any]) -> Optional[int]:
"""NIP-40, or None. The first well-formed expiration tag wins; a value that
is not ASCII digits is not an expiration tag at all, so it is skipped
rather than read as expiry now: a peer cannot be allowed to evict an event
by sending garbage, and two implementations with different int parsers
cannot be allowed to disagree about whether it was garbage."""
for t in tags(event, 'expiration'):
if len(t) > 1:
ts = ascii_int(t[1])
if ts is not None:
return ts
return None
[docs]def supersedes(new: Mapping[str, Any], old: Mapping[str, Any]) -> bool:
"""NIP-01: later created_at wins; on a tie the lexicographically lower id.
The tie-break matters more than it looks. Two nodes that saw both versions
must pick the same one or their fingerprints differ forever, and a tie is
exactly what a replaceable event republished within one second produces.
"""
if new['created_at'] != old['created_at']:
return new['created_at'] > old['created_at']
return new['id'] < old['id']
# ------------------------------------------------------------ NIP-01 filter --
[docs]def matches(event: Mapping[str, Any], filt: Mapping[str, Any]) -> bool:
"""NIP-01 filter matching, including single-letter #tag filters.
Every condition is an AND across fields and an OR within a field, which is
NIP-01's rule and the one thing a backend cannot get creative about.
"""
if 'ids' in filt and event['id'] not in filt['ids']:
return False
if 'authors' in filt and event['pubkey'] not in filt['authors']:
return False
if 'kinds' in filt and event['kind'] not in filt['kinds']:
return False
if 'since' in filt and event['created_at'] < filt['since']:
return False
if 'until' in filt and event['created_at'] > filt['until']:
return False
for key, wanted in filt.items():
if not key.startswith('#') or len(key) != 2:
continue
name = key[1]
values = {t[1] for t in event.get('tags', []) if len(t) > 1 and t[0] == name}
if values.isdisjoint(wanted):
return False
return True
_SCALARS = (str, int, float, bool, type(None))
# What a compiled filter holds under a key: a set of ids, authors, kinds or
# tag values; a list left as it came (one holding something unhashable); or
# a scalar, such as a search term.
FilterValue = Union[frozenset[Union[str, int, float, bool, None]], Sequence[Any], int, str, None]
[docs]def compiled(filt: Mapping[str, Any]) -> Mapping[str, FilterValue]:
"""`filt` without since/until/limit, its lists as sets: the form `matches`
answers fastest when it is asked about many events.
`_in_range` and `newest_matching` built that reduced filter again for
every event, and `matches` tested `ids` and `authors` as lists - O(n) per
event, with up to validate.MAX_IDS ids - and rebuilt each tag set per event
per tag (audit 2026-10-15, E2). Same answers: membership is membership. A
list holding anything but scalars is left a list, and answered as before.
"""
out: dict[str, FilterValue] = {}
for k, v in filt.items():
if k in ('since', 'until', 'limit'):
continue
setlike = (k in ('ids', 'authors', 'kinds') or k.startswith('#')) and isinstance(v, (list, tuple))
out[k] = frozenset(v) if setlike and all(isinstance(x, _SCALARS) for x in v) else v
return out
[docs]class MeshStore:
"""The interface, and every rule that is the same whatever the backend.
A backend supplies raw storage: _put, _drop, _all, _get, _tombstone,
_is_tombstoned. Everything that decides set membership lives here, so a new
backend cannot accidentally invent its own answer to the four questions
above - which is the whole point of a shared suite.
"""
# ---- backend surface -------------------------------------------------
def _put(self, event: Mapping[str, Any]) -> None:
raise NotImplementedError
def _drop(self, event_id: str) -> None:
raise NotImplementedError
def _all(self) -> list[Mapping[str, Any]]:
raise NotImplementedError
def _get(self, event_id: str) -> Optional[Mapping[str, Any]]:
raise NotImplementedError
# Narrowing is optional: a backend that cannot do better than reading
# everything inherits these and is still correct. A backend that can - one
# with an index - overrides them, and the shared suite proves the two give
# the same answers.
#
# This is not an optimisation for its own sake. Every scan below was
# `self._all()`, which materialises the whole store as Python dicts, so a
# node paid O(store) on every query AND on every replaceable insert. On a
# node with a public face taking about 150 events an hour it cost roughly
# 50 MB of RSS an hour against an 8,000-event store, because CPython does
# not return freed arenas to the system, and only a restart brought it
# back. A relay that degrades quadratically in the size of its own shard is
# a relay that cannot hold a large one.
def _narrow(self, lo: int, hi: int, kinds: Optional[Sequence[int]] = None,
authors: Optional[Sequence[str]] = None,
tags: Optional[Mapping[str, Sequence[str]]] = None,
ids: Optional[Sequence[str]] = None) -> list[Mapping[str, Any]]:
"""Everything in [lo, hi], optionally of these kinds, authors, tags or ids.
Narrowing may return a superset and must never return less: `matches`
above decides membership, and this only says where to look. A backend
that narrows none of it is still correct, which is what these defaults
are for.
"""
out = [e for e in self._all() if lo <= e['created_at'] <= hi]
if kinds is not None:
ks = set(kinds)
out = [e for e in out if e['kind'] in ks]
if authors is not None:
aus = set(authors)
out = [e for e in out if e['pubkey'] in aus]
if ids is not None:
want_ids = set(ids)
out = [e for e in out if e['id'] in want_ids]
for name, values in (tags or {}).items():
want = set(values)
out = [e for e in out
if any(len(t) > 1 and t[0] == name and t[1] in want for t in e.get('tags', []))]
return out
def _by_author_kind(self, pubkey: str, kind: int) -> list[Mapping[str, Any]]:
"""Every event one author wrote of one kind. Order is not promised."""
return [e for e in self._all() if e['pubkey'] == pubkey and e['kind'] == kind]
def _by_author(self, pubkey: str) -> list[Mapping[str, Any]]:
"""Every event one author wrote. Order is not promised."""
return [e for e in self._all() if e['pubkey'] == pubkey]
def _tombstone(self, key: str, deleter: str) -> None:
raise NotImplementedError
def _tombstoned_by(self, key: str) -> set[str]:
raise NotImplementedError
def _atomic(self) -> ContextManager[None]:
"""One transaction around a decision that writes more than once.
Supersession drops the old version and puts the new one; a deletion
drops each target, tombstones it, and stores itself. Each primitive
used to commit on its own, so a process that died between two of them
left a coordinate with no winner, or a dropped event with no tombstone
- which the next peer to offer it would resurrect (audit 2026-10-15,
B4). A backend with transactions overrides this so `insert_event` is
all or nothing. The default is for a backend with nothing to roll back.
"""
return contextlib.nullcontext()
# ---- interface -------------------------------------------------------
[docs] def get_event(self, event_id: str) -> Optional[Mapping[str, Any]]:
return self._get(event_id)
[docs] def has_event(self, event_id: str) -> bool:
return self._get(event_id) is not None
[docs] def insert_event(self, event: Mapping[str, Any], now: int) -> str:
"""Returns one of: stored, duplicate, ephemeral, expired, superseded,
deleted, or a deletion outcome. `now` is explicit rather than read from
the clock so that two backends fed the same history agree exactly.
All or nothing, through `_atomic`: every write one insertion makes
lands together or not at all.
"""
with self._atomic():
return self._insert_event(event, now)
def _insert_event(self, event: Mapping[str, Any], now: int) -> str:
eid = event['id']
# A deletion may have arrived first. Its authority could not be checked
# then, because that needs this event's author; it is checked now.
# A kind 5 is never deleted, so a tombstone naming one is ignored
# in either arrival order.
if event['kind'] != 5 and event['pubkey'] in self._tombstoned_by(eid):
return 'deleted'
coord = coordinate(event)
if coord is not None and self._deleted_by_coordinate(event, coord):
return 'deleted'
if self._get(eid) is not None:
return 'duplicate'
if is_ephemeral(event['kind']):
return 'ephemeral'
exp = expiration(event)
if exp is not None and exp <= now:
return 'expired'
if event['kind'] == 5:
return self._apply_deletion(event)
if coord is not None:
# A coordinate is (kind, pubkey, d), so the author and the kind
# narrow it to a handful before anything is decoded and compared.
for other in list(self._by_author_kind(event['pubkey'], event['kind'])):
if coordinate(other) == coord:
if supersedes(event, other):
self._drop(other['id'])
else:
return 'superseded'
self._put(event)
return 'stored'
def _deleted_by_coordinate(self, event: Mapping[str, Any], coord: str) -> bool:
"""NIP-09 by coordinate, checked on arrival: a kind 5 the same author
signed, naming this coordinate in an `a` tag, and created no earlier
than the event. The kind 5 is stored, so it is the record; a coordinate
is a name for a *series* and a later version of the series is new
content the deletion did not speak about."""
return any(coord in _coordinates(d) and event['created_at'] <= d['created_at']
for d in self._by_author_kind(event['pubkey'], 5))
def _apply_deletion(self, deletion: Mapping[str, Any]) -> str:
"""NIP-09, with the authority check the specification makes normative:
a deletion may only remove events its own author signed. The kind 5 is
itself stored, because a peer that never receives it never learns of
the deletion, and tombstones are kept so a later re-offer of the same
id cannot resurrect it.
An `a` tag is a NIP-01 coordinate, `kind:pubkey:d`, and reaches only
versions created at or before the deletion. An `a` tag whose pubkey is
not the deleter's names nothing it has authority over and is ignored.
A kind 5 cannot itself be deleted: a deletion of a deletion has no
effect, or a peer could be made to forget what it was told to forget."""
removed = 0
targets = {t[1] for t in deletion.get('tags', []) if len(t) > 1 and t[0] == 'e'}
coords = _coordinates(deletion)
# Only the deleter's own events can be affected - the authority check
# below is normative, not an optimisation - so that is the set to read.
for other in list(self._by_author(deletion['pubkey'])):
if other['kind'] == 5:
continue # a deletion is never deleted
hit = other['id'] in targets or (coordinate(other) in coords
and other['created_at'] <= deletion['created_at'])
if not hit:
continue
if other['pubkey'] != deletion['pubkey']:
continue # a stranger's deletion is not authority over this event
self._drop(other['id'])
self._tombstone(other['id'], deletion['pubkey'])
removed += 1
# Targets not held yet. Recording the deleter's key is what lets the
# authority check happen on arrival instead of being skipped: a
# deletion that names a stranger's event tombstones nothing it can
# enforce, whichever order the two events turn up in. Coordinates need
# no tombstone: the stored kind 5 is consulted on arrival.
for t in targets:
if self._get(t) is None:
self._tombstone(t, deletion['pubkey'])
self._put(deletion)
return 'deleted %d' % removed
# ---- range queries ---------------------------------------------------
def _in_range(self, filt: Mapping[str, Any], since: int, until: int, now: int) -> list[Mapping[str, Any]]:
"""The set. `since`/`until` bound the time axis and a filter's own
since/until narrows further - the intersection, never the wider of the
two, so that neither argument can silently widen the other."""
lo, hi = since, until
if 'since' in filt:
lo = max(lo, filt['since'])
if 'until' in filt:
hi = min(hi, filt['until'])
out: list[Mapping[str, Any]] = []
# Narrowed by the backend where it can be, then decided here. The time
# bound is the one narrowing every caller supplies, and it is the one
# the events table is indexed on.
# NIP-01 indexes single-letter tags and only those, which is exactly
# the set a backend can be asked to narrow on.
want_tags = {k[1:]: v for k, v in filt.items()
if len(k) == 2 and k[0] == '#' and isinstance(v, (list, tuple))}
rest = compiled(filt)
for ev in self._narrow(lo, hi, filt.get('kinds'), filt.get('authors'), want_tags, filt.get('ids')):
exp = expiration(ev)
if exp is not None and exp <= now:
continue
if not matches(ev, rest):
continue
out.append(ev)
return out
[docs] def enumerate_range(self, filt: Mapping[str, Any], since: int, until: int, now: int) -> list[str]:
"""(created_at ASC, id ASC). The order is observable, so it is normative
even though the fingerprint does not need it."""
return [e['id'] for e in sorted(self._in_range(filt, since, until, now),
key=lambda e: (e['created_at'], e['id']))]
[docs] def items_range(self, filt: Mapping[str, Any], since: int, until: int, now: int) -> list[tuple[int, str]]:
"""(created_at, id) pairs in the same order, which is the record shape
negentropy reconciles over - timestamp first, id second, ascending."""
return [(e['created_at'], e['id'])
for e in sorted(self._in_range(filt, since, until, now),
key=lambda e: (e['created_at'], e['id']))]
[docs] def fingerprint_range(self, filt: Mapping[str, Any], since: int, until: int, now: int) -> str:
"""NIP-77's fingerprint over that set. 16 bytes, as 32 hex characters."""
return replication.fingerprint(self.enumerate_range(filt, since, until, now))
[docs] def newest_matching(self, filt: Mapping[str, Any], since: int, until: int,
now: int, cap: int) -> list[Mapping[str, Any]]:
"""At most `cap` matching events, newest first, ties broken by id ascending.
Every other range query answers "the whole set", which is the only
answer a fingerprint or a reconciliation can use. A REQ is the one
caller that knows up front how many it will send, and on an open face
the set can be four orders of magnitude larger than the limit. Asking
for the set and then slicing it is what a backend cannot afford, so
the question is asked in the shape the answer is wanted in.
"""
ordered = sorted(self._in_range(filt, since, until, now),
key=lambda e: (-e['created_at'], e['id']))
return ordered[:max(0, cap)]
[docs]class MemoryStore(MeshStore):
"""A dict. The naive backend the specification says is conformant."""
[docs] def __init__(self) -> None:
self._events: dict[str, Mapping[str, Any]] = {}
self._tombs: dict[str, set[str]] = {}
def _put(self, event: Mapping[str, Any]) -> None:
self._events[event['id']] = dict(event)
def _drop(self, event_id: str) -> None:
self._events.pop(event_id, None)
def _all(self) -> list[Mapping[str, Any]]:
return list(self._events.values())
def _get(self, event_id: str) -> Optional[Mapping[str, Any]]:
return self._events.get(event_id)
def _tombstone(self, key: str, deleter: str) -> None:
self._tombs.setdefault(key, set()).add(deleter)
def _tombstoned_by(self, key: str) -> set[str]:
return self._tombs.get(key, set())
[docs]class SqliteStore(MeshStore):
"""sqlite3 from the standard library, storing the event as canonical JSON.
Deliberately naive: it scans. The specification is explicit that how a
backend computes a range is unspecified and only the value on the wire is
normative, so a backend that scans and one with a prefix-sum index must
agree - and this one exists to be the slow, obviously-correct side of that
comparison.
"""
[docs] def __init__(self, path: str = ':memory:') -> None:
self.db = sqlite3.connect(path)
self.db.executescript(
'CREATE TABLE IF NOT EXISTS events ('
' id TEXT PRIMARY KEY, pubkey TEXT, created_at INTEGER, kind INTEGER, json TEXT);'
'CREATE INDEX IF NOT EXISTS events_created_at ON events (created_at, id);'
# Supersession asks for one author's events of one kind on every
# replaceable insert, and a deletion asks for one author's events.
# Without this both are a table scan, which is what they were.
'CREATE INDEX IF NOT EXISTS events_author_kind ON events (pubkey, kind);'
'CREATE TABLE IF NOT EXISTS tombstones ('
' key TEXT, deleter TEXT, PRIMARY KEY (key, deleter));'
# NIP-01 indexes single-letter tags and only those, so that is what
# is kept here. Without it a client asking for its own gift wraps -
# kinds 1059, #p its key - decoded every wrap in the store to find
# none, which on a public face was half a second and eight
# megabytes per subscription, repeated per client.
'CREATE TABLE IF NOT EXISTS tags ('
' id TEXT, name TEXT, value TEXT, PRIMARY KEY (id, name, value)) WITHOUT ROWID;'
'CREATE INDEX IF NOT EXISTS tags_name_value ON tags (name, value);')
self.db.commit()
self._backfill_tags()
def _backfill_tags(self) -> None:
"""Index the tags of events stored before this table existed.
A store opened by an older build has events and no rows here, and an
empty index is indistinguishable from an index over events with no
tags - so the question asked is whether any event is missing from it,
not whether the table is empty.
"""
missing = self.db.execute(
'SELECT json FROM events WHERE id NOT IN (SELECT id FROM tags)').fetchall()
if not missing:
return
for (raw,) in missing:
self._index_tags(json.loads(raw))
self.db.commit()
def _index_tags(self, event: Mapping[str, Any]) -> None:
rows = {(event['id'], t[0], t[1]) for t in event.get('tags', [])
if len(t) > 1 and len(t[0]) == 1 and isinstance(t[1], str)}
# An event with no single-letter tag still gets one row, with a null
# name, or _backfill_tags would rediscover it as missing on every open.
self.db.executemany('INSERT OR REPLACE INTO tags VALUES (?,?,?)',
rows or [(event['id'], '', '')])
# Nesting depth of `_atomic`. A class attribute so a subclass that opens
# the connection itself, and skips this __init__, starts at zero too.
_atomic_depth: int = 0
def _commit(self) -> None:
"""Commit, unless an `_atomic` block will commit for us."""
if self._atomic_depth == 0:
self.db.commit()
@contextlib.contextmanager
def _atomic(self) -> Iterator[None]:
"""BEGIN on the outermost block; COMMIT if it returns, ROLLBACK if it
raises. Inner blocks join the outer one. BEGIN is explicit so this
holds whatever isolation mode the connection was opened in."""
outer = self._atomic_depth == 0
if outer and not self.db.in_transaction:
self.db.execute('BEGIN')
self._atomic_depth += 1
try:
yield
except BaseException:
self._atomic_depth -= 1
if outer:
self.db.rollback()
raise
self._atomic_depth -= 1
if outer:
self.db.commit()
def _put(self, event: Mapping[str, Any]) -> None:
self.db.execute('INSERT OR REPLACE INTO events VALUES (?,?,?,?,?)',
(event['id'], event['pubkey'], event['created_at'], event['kind'],
crypto.canonical(event).decode()))
self._index_tags(event)
self._commit()
def _drop(self, event_id: str) -> None:
self.db.execute('DELETE FROM events WHERE id = ?', (event_id,))
self.db.execute('DELETE FROM tags WHERE id = ?', (event_id,))
self._commit()
def _all(self) -> list[Mapping[str, Any]]:
return [json.loads(r[0]) for r in self.db.execute('SELECT json FROM events')]
def _where(self, lo: int, hi: int, kinds: Optional[Sequence[int]] = None,
authors: Optional[Sequence[str]] = None,
tags: Optional[Mapping[str, Sequence[str]]] = None,
ids: Optional[Sequence[str]] = None
) -> Optional[tuple[str, list[Any]]]:
# A coverage claim fingerprints a shard over an unbounded range, so the
# time bound alone left it reading the whole store to answer with 38
# ids. kind and author are columns; the tag it also filters on is not,
# and stays in `matches`. None means no row can match.
where: list[str] = ['created_at BETWEEN ? AND ?']
args: list[Any] = [int(lo), int(hi)]
if kinds is not None:
ks = list(kinds)
if not ks:
return None
sql, vals = in_clause('kind', [int(k) for k in ks])
where.append(sql)
args.extend(vals)
if authors is not None:
aus = list(authors)
if not aus:
return None
sql, vals = in_clause('pubkey', aus)
where.append(sql)
args.extend(vals)
# A REQ by id read every event in the time range, newest first, until
# one matched: 36 ms for one missing id on 20,000 events, against
# 0.03 ms by primary key, under the store lock and on demand for any
# client (audit 2026-10-15, E1). An id that is not a string can never
# equal one, so dropping it changes no answer.
if ids is not None:
wanted = [i for i in ids if isinstance(i, str)]
if not wanted:
return None
sql, vals = in_clause('id', wanted)
where.append(sql)
args.extend(vals)
# One EXISTS per tag name, which is NIP-01's own semantics: values
# within a name are OR, and different names are AND.
for name, values in (tags or {}).items():
vs = list(values)
if not vs:
return None
sql, vals = in_clause('tags.value', vs)
where.append('EXISTS (SELECT 1 FROM tags WHERE tags.id = events.id '
'AND tags.name = ? AND %s)' % sql)
args.append(name)
args.extend(vals)
return ' AND '.join(where), args
def _narrow(self, lo: int, hi: int, kinds: Optional[Sequence[int]] = None,
authors: Optional[Sequence[str]] = None,
tags: Optional[Mapping[str, Sequence[str]]] = None,
ids: Optional[Sequence[str]] = None) -> list[Mapping[str, Any]]:
clause = self._where(lo, hi, kinds, authors, tags, ids)
if clause is None:
return []
return [json.loads(r[0]) for r in self.db.execute(
'SELECT json FROM events WHERE ' + clause[0], clause[1])]
[docs] def newest_matching(self, filt: Mapping[str, Any], since: int, until: int,
now: int, cap: int) -> list[Mapping[str, Any]]:
"""The bounded answer, read newest first and stopped at `cap`.
The inherited version narrows in SQL and then holds every match as a
parsed event so it can be sorted. That is the whole store on an open
face - which is how a 1 GB box serving 8,923 events died on a REQ for
the newest twenty. Here sqlite does the ordering on the index the
events table already has, and rows are parsed one at a time until
enough of them have matched.
"""
if cap <= 0:
return []
lo, hi = since, until
if 'since' in filt:
lo = max(lo, filt['since'])
if 'until' in filt:
hi = min(hi, filt['until'])
want_tags = {k[1:]: v for k, v in filt.items()
if len(k) == 2 and k[0] == '#' and isinstance(v, (list, tuple))}
clause = self._where(lo, hi, filt.get('kinds'), filt.get('authors'), want_tags, filt.get('ids'))
if clause is None:
return []
rest = compiled(filt)
out: list[Mapping[str, Any]] = []
cursor = self.db.execute(
'SELECT json FROM events WHERE ' + clause[0] + ' ORDER BY created_at DESC, id ASC',
clause[1])
for row in cursor:
ev = json.loads(row[0])
exp = expiration(ev)
if exp is not None and exp <= now:
continue
if not matches(ev, rest):
continue
out.append(ev)
if len(out) >= cap:
break
return out
def _by_author_kind(self, pubkey: str, kind: int) -> list[Mapping[str, Any]]:
return [json.loads(r[0]) for r in self.db.execute(
'SELECT json FROM events WHERE pubkey = ? AND kind = ?', (pubkey, int(kind)))]
def _by_author(self, pubkey: str) -> list[Mapping[str, Any]]:
return [json.loads(r[0]) for r in self.db.execute(
'SELECT json FROM events WHERE pubkey = ?', (pubkey,))]
def _get(self, event_id: str) -> Optional[Mapping[str, Any]]:
r = self.db.execute('SELECT json FROM events WHERE id = ?', (event_id,)).fetchone()
return json.loads(r[0]) if r else None
def _tombstone(self, key: str, deleter: str) -> None:
self.db.execute('INSERT OR IGNORE INTO tombstones VALUES (?,?)', (key, deleter))
self._commit()
def _tombstoned_by(self, key: str) -> set[str]:
return {r[0] for r in self.db.execute(
'SELECT deleter FROM tombstones WHERE key = ?', (key,))}
[docs] def close(self) -> None:
"""Safely close the underlying database connection."""
if hasattr(self, 'db') and self.db is not None:
try:
self.db.close()
except Exception:
pass
def __del__(self) -> None:
try:
self.close()
except Exception:
pass
[docs]def event_id(pubkey: str, created_at: int, kind: int, tags: Optional[Sequence[Sequence[str]]] = None, content: str = '') -> str:
"""NIP-01 event id: sha256 over the canonical serialisation."""
tags = tags or []
ser = json.dumps([0, pubkey, created_at, kind, tags, content],
separators=(',', ':'), ensure_ascii=False)
return hashlib.sha256(ser.encode()).hexdigest()
[docs]def make_event(pubkey: str, created_at: int, kind: int, tags: Optional[Sequence[Sequence[str]]] = None, content: str = '') -> UnsignedNostrEvent:
"""A well-formed event without a signature.
The store never checks one: the interface takes "an independently
authenticated event", so verification happens before insertion and belongs
to whatever did the authenticating. That makes this cheap enough to build
corpora of thousands, which the tests need.
It is *not* a complete nostr event - NIP-01 requires a sig, and anything
that puts an event on the wire needs make_signed_event instead.
"""
tag_list: TagList = [list(t) for t in (tags or [])]
return {'id': event_id(pubkey, created_at, kind, tag_list, content), 'pubkey': pubkey,
'created_at': created_at, 'kind': kind, 'tags': tag_list, 'content': content}
[docs]def make_signed_event(sk_hex: str, created_at: int, kind: int, tags: Optional[Sequence[Sequence[str]]] = None, content: str = '') -> NostrEvent:
"""A complete NIP-01 event, signature and all.
Slow - the curve arithmetic here is pure Python - so it is for the places
that need a real event rather than a corpus: anything validated against
schemas/common/nostr-event.schema.json, or put in a sync/1.0/deliver.
"""
from ..crypto import crypto
unsigned = make_event(crypto.pubkey(sk_hex), created_at, kind, tags, content)
# Built key by key rather than by copying the unsigned dict: a TypedDict
# with a required `sig` cannot be made from one without it, and the
# checker holds us to that.
return {'id': unsigned['id'], 'pubkey': unsigned['pubkey'],
'created_at': unsigned['created_at'], 'kind': unsigned['kind'],
'tags': unsigned['tags'], 'content': unsigned['content'],
'sig': crypto.schnorr_sign(bytes.fromhex(unsigned['id']), sk_hex)}
[docs]def verify_event(ev: Mapping[str, Any]) -> bool:
"""NIP-01: the id is the hash of the serialised event and the signature is
BIP-340 over that id by the event's pubkey. False for anything malformed,
which includes a missing `tags` or `content`: NIP-01 requires both, and an
event without them is not one that another implementation would hash the
same way."""
from ..crypto import crypto
from ..wire.wire_types import is_event_shaped
if not is_event_shaped(ev) or not is_hex64(ev['pubkey']) or not is_hex64(ev['id']) or ev['created_at'] < 0:
return False
try:
want = event_id(ev['pubkey'], ev['created_at'], ev['kind'], ev['tags'], ev['content'])
return ev['id'] == want and crypto.schnorr_verify(bytes.fromhex(want), ev['pubkey'], ev['sig'])
except (KeyError, ValueError, TypeError, AttributeError):
return False