Source code for fleetmesh_node.storage.base

"""The backend-independent half of a store: the lock, the transaction, and the
declarative schema.
Bound to `SqliteDialect` by default. Everything here is backend-independent
and tested across supported backends.

Subclasses declare their schema rather than running DDL in ``__init__``:

- ``SCHEMA``: ``CREATE TABLE IF NOT EXISTS`` / ``CREATE INDEX IF NOT EXISTS``
  statements, one per entry, executed in order on every open. They must be
  idempotent, which the ``IF NOT EXISTS`` form guarantees.
- ``MIGRATIONS``: ``(version, sql)`` pairs, ascending. Each is applied once,
  when the stored schema version is below ``version``; afterwards the version
  is raised to the highest applied. A fresh database runs through every
  migration too, so ``SCHEMA`` can stay the original DDL and a column added
  later is expressed exactly once, as the ``ALTER TABLE`` that added it. A
  database from before the version was tracked may already carry such a column
  (the old open path added it under a ``has_column`` guard): a duplicate-column
  error from an ``ADD COLUMN`` migration is therefore treated as "already
  applied", which the dialect decides rather than a message match here.
- ``_post_schema()``: hook for data backfills that need Python, run inside the
  same transaction after the migrations.

All of it runs in ``_ensure_schema()`` as one transaction at open, so a crash
midway leaves the database as it was.

``DURABILITY`` names what a store needs of its backend, for the three classes
in docs/storage-backends.md section 6:

    'commit'  a write is durable before the call returns
    'window'  may be batched, durable before the window closes or before a
              receipt citing it leaves
    'cache'   may be lost; rebuildable from elsewhere

It is declared, not yet enforced: on SQLite every write is already a commit,
so the classes describe the contract a networked backend will have to honour.
I15 is the defect that exists because this was never named - see
docs/audit-2026-10-07-4-issues.md.
"""
from __future__ import annotations

import contextlib
import threading
import time
from typing import Any, Callable, Iterator, Optional, Sequence, TypedDict, Union

from ..compartment.schema import Migration
from .dialect import Dialect, SqliteDialect
from .locator import redact

LEASE_SECONDS: int = 900


[docs]class StoreLease(TypedDict): """The lease as `meshnode health` reports it. A named record rather than a loose mapping, because check-dicts.py holds this tree to typing what it returns, and a row an operator reads by field name is exactly the case that rule exists for. """ owner: str expires_at: int mine: bool expires_in: int
[docs]class StoreOwnedError(RuntimeError): """Another node holds a live lease on this database. Raised by `open()` rather than letting two nodes write one database. One node per database is a decision (docs/storage-backends.md header), and it is a decision a connection string can break where a file could not: `meters` is keyed (peer, epoch, side) and `grants` (peer, direction), and neither includes the node, so two nodes would interleave rows, read each other's accounting as their own, and price windows that never happened with nothing in the schema complaining. """
class _Cursor: """A cursor whose SQL goes through the dialect on the way out. `transaction()` used to yield the driver's own cursor, so every `cur.execute(...)` inside a `with store.transaction() as cur:` block circumvented the translation the base exists to apply - 37 statements across the tree, invisible while the SQLite dialect's `sql()` was the identity function. Handing out something that bypasses your own interface is the flaw; counting the call sites was the symptom. Everything else - fetchone, fetchall, rowcount, description - passes straight through to the real cursor. """ __slots__ = ('_cur', '_dialect') def __init__(self, cur: Any, dialect: Dialect) -> None: self._cur = cur self._dialect = dialect def execute(self, q: str, args: Sequence[Any] = ()) -> Any: return self._cur.execute(self._dialect.sql(q), tuple(args)) def executemany(self, q: str, seq: Any) -> Any: return self._cur.executemany(self._dialect.sql(q), seq) def __iter__(self) -> Any: return iter(self._cur) def __getattr__(self, name: str) -> Any: return getattr(self._cur, name)
[docs]class BaseStore: """Thread-safe persistence with a declarative schema, over any dialect.""" SCHEMA: tuple[Any, ...] = () MIGRATIONS: tuple[Union[Migration, tuple[int, str]], ...] = () DURABILITY: str = 'commit'
[docs] def __init__(self, path: str = ':memory:', timeout: float = 5.0, dialect: Optional[Dialect] = None, owner: Optional[str] = None, lease_seconds: int = LEASE_SECONDS, clock: Optional[Callable[[], int]] = None, readonly: bool = False, schema: Optional[Sequence[Any]] = None) -> None: self.path: str = path self.dialect: Dialect = dialect or SqliteDialect() self._lock: threading.RLock = threading.RLock() self.owner: Optional[str] = owner self.lease_seconds: int = int(lease_seconds) self.clock: Callable[[], int] = clock or (lambda: int(time.time())) self.readonly: bool = readonly if schema is not None: self.SCHEMA = tuple(schema) self.db: Any = self.dialect.connect(path, timeout, readonly) if not readonly: self.dialect.tune(self.db, path, self._lock) # Claimed before the schema runs: migrations are writes, and a # database another node owns is not this node's to migrate. if owner is not None: self._claim_lease(owner) self._ensure_schema()
# ---- One node per database ------------------------------------------------ def _claim_lease(self, owner: str) -> None: """Take the lease, or refuse because another node holds a live one. A stale lease is takeable, which is what makes an ordinary restart work: a node that died holds a lease nobody renews, and the next start - its own or a replacement's - takes it once it expires. """ now = self.clock() with self._lock: self.db.execute(self.dialect.sql( 'CREATE TABLE IF NOT EXISTS fleetmesh_owner (' ' only_row INTEGER PRIMARY KEY, owner TEXT NOT NULL, expires_at INTEGER NOT NULL)')) cur = self.db.execute(self.dialect.sql( 'SELECT owner, expires_at FROM fleetmesh_owner WHERE only_row = 1')) try: row = cur.fetchone() finally: cur.close() if row is not None and row[0] != owner and int(row[1]) > now: raise StoreOwnedError( 'database %s is held by node %s until %d (now %d); one node per database' % (self.safe_path, str(row[0])[:16], int(row[1]), now)) self.db.execute(self.dialect.sql( 'INSERT OR REPLACE INTO fleetmesh_owner VALUES (1, ?, ?)'), (owner, now + self.lease_seconds)) self.db.commit()
[docs] def renew_lease(self) -> None: """Push the lease out. The tick calls this; a node that stops renewing releases the database to its successor once the lease runs out.""" if self.owner is None: return with self._lock: self.db.execute(self.dialect.sql( 'INSERT OR REPLACE INTO fleetmesh_owner VALUES (1, ?, ?)'), (self.owner, self.clock() + self.lease_seconds)) self.db.commit()
[docs] def lease(self) -> Optional[StoreLease]: """The lease as health should report it, or None when none is held.""" with self._lock: if not self.dialect.has_table(self.db, 'fleetmesh_owner'): return None cur = self.db.execute(self.dialect.sql( 'SELECT owner, expires_at FROM fleetmesh_owner WHERE only_row = 1')) try: row = cur.fetchone() finally: cur.close() if row is None: return None return StoreLease(owner=str(row[0]), expires_at=int(row[1]), mine=row[0] == self.owner, expires_in=int(row[1]) - self.clock())
# ---- Declarative Schema ------------------------------------------------- @property def safe_path(self) -> str: """`path` with any credentials removed, for a message or a log. A SQLite target is a filename and comes back unchanged. A server target is a connection string and carries a password, so anything an operator reads - a refusal, a log line, a health block - must use this and not `path`. The lease refusal below is the case that found it: it is the one message a misconfigured fleet is guaranteed to produce, and it printed the whole URL. """ return redact(self.path) if '://' in self.path else self.path
[docs] def schema_name(self) -> str: """How this store identifies itself to the dialect's version table. One node per database means all fourteen stores share one database on a server backend, where a single version row would make their separate migration series collide. """ return type(self).__name__
[docs] def user_version(self) -> int: """This store's stored schema version (0 if it never migrated).""" with self._lock: return self.dialect.get_version(self.db, self.schema_name())
def _post_schema(self) -> None: """Data backfill hook, run inside the schema transaction. No-op by default.""" def _ensure_schema(self) -> None: """Apply ``SCHEMA`` then any pending ``MIGRATIONS`` in one transaction.""" if not self.SCHEMA and not self.MIGRATIONS: return with self._lock: self.dialect.begin(self.db) try: for entry in self.SCHEMA: if hasattr(entry, 'to_sql'): sql_stmt = entry.to_sql(dialect=self.dialect.name) if hasattr(self.dialect, 'learn_table'): self.dialect.learn_table(entry) self.db.execute(self.dialect.sql(sql_stmt)) if hasattr(entry, 'secondary_sql'): for sec in entry.secondary_sql(dialect=self.dialect.name): self.db.execute(self.dialect.sql(sec)) else: self.db.execute(self.dialect.sql(entry)) current = self.dialect.get_version(self.db, self.schema_name()) target = current for m in self.MIGRATIONS: if isinstance(m, Migration): version, sql = m.to_tuple() else: version, sql = m if version <= current: continue try: self.db.execute(self.dialect.sql(sql)) except BaseException as e: if not self.dialect.is_duplicate_column(e): raise target = max(target, version) if target != current: self.dialect.set_version(self.db, self.schema_name(), target) self._post_schema() self.dialect.commit(self.db) except BaseException: self.dialect.rollback(self.db) raise
[docs] def integrity_check(self) -> Optional[str]: """None if the database is sound, else what is wrong with it.""" with self._lock: return self.dialect.integrity_check(self.db)
[docs] def vacuum(self, pages: int = 50) -> int: """Reclaim freed space and refresh planner statistics, per the dialect.""" with self._lock: return self.dialect.vacuum(self.db, pages)
[docs] def close(self) -> None: """Safely close the underlying database connection.""" db = getattr(self, 'db', None) if db is None: return self.db = None lock = getattr(self, '_lock', None) if lock is not None: try: with lock: try: db.close() except Exception: pass except Exception: try: db.close() except Exception: pass else: try: db.close() except Exception: pass
def __del__(self) -> None: try: self.close() except Exception: pass # ---- Query Execution with Guaranteed Cursor Cleanup --------------------- def _one(self, q: str, args: Sequence[Any] = ()) -> Optional[tuple[Any, ...]]: """Execute query and fetch one row, closing the cursor before releasing the lock. Prevents InterfaceError/SQLITE_MISUSE from unfinalized cursors collected by GC. """ with self._lock: cur = self.db.execute(self.dialect.sql(q), tuple(args)) try: return cur.fetchone() finally: cur.close() def _all(self, q: str, args: Sequence[Any] = ()) -> list[tuple[Any, ...]]: """Execute query and fetch all rows, closing the cursor before releasing the lock.""" with self._lock: cur = self.db.execute(self.dialect.sql(q), tuple(args)) try: return cur.fetchall() finally: cur.close()
[docs] def execute(self, q: str, args: Sequence[Any] = ()) -> Any: """Execute a single statement and return the cursor within the locked context.""" with self._lock: return self.db.execute(self.dialect.sql(q), tuple(args))
[docs] def executescript(self, script: str) -> None: """Execute multiple SQL statements and commit.""" with self._lock: self.db.executescript(self.dialect.sql(script)) self.db.commit()
[docs] def commit(self) -> None: """Commit active transaction under lock.""" with self._lock: self.db.commit()
[docs] @contextlib.contextmanager def transaction(self) -> Iterator[Any]: """Atomic block: commit on success, rollback on exception. The retry a serializable backend would need belongs here and nowhere else, which is what keeps CockroachDB a later addition rather than a rewrite (docs/storage-backends.md section 5). None is built: SQLite never raises a retryable error, and neither does a Postgres backend that keeps this one-connection-behind-one-lock shape. """ with self._lock: cur = self.db.cursor() try: yield _Cursor(cur, self.dialect) self.db.commit() except Exception: self.db.rollback() raise finally: cur.close()
# ---- Schema Reflection Helpers ------------------------------------------
[docs] def has_table(self, table_name: str) -> bool: """Return True if the specified table exists in the database.""" with self._lock: return self.dialect.has_table(self.db, table_name)
[docs] def has_column(self, table_name: str, column_name: str) -> bool: """Return True if the column exists in the table schema.""" with self._lock: return self.dialect.has_column(self.db, table_name, column_name)