Source code for fleetmesh_node.protocols.rhio.store

"""Rhio blob storage and Bao-compatible chunked tree hashing.

Standard library only. Provides:
- Bao chunked Merkle tree verification (1024-byte chunks).
- Local CAS blob repository and SQLite synchronization metadata.
- S3 / MinIO synchronization state tracking.
"""
from __future__ import annotations

import base64
import binascii
import hashlib
import json
import os
import re
import shutil
import threading
import time
import urllib.parse
from typing import Any, Callable, Mapping, Optional, Sequence, TypedDict

from fleetmesh_core.conformance.errors import FleetmeshError
from fleetmesh_core.textile import bao as _bao

__all__ = [
    'FIBER_KINDS',
    'FiberAccessError',
    'INTERACTION_KINDS',
    'MAX_FIBER_BYTES',
    'MAX_OWNER_BYTES',
    'MAX_UPLOAD_BODY_BYTES',
    'PAYLOAD_KINDS',
    'RhioBaoTree',
    'RhioBlobRecord',
    'RhioBlobSync',
    'RhioStoreError',
    'VISIBILITIES',
    'binary_http_response',
    'handle_fiber_http',
    'is_fiber_route',
]


from fleetmesh_node.compartment.schema import Column, Index, Table
from fleetmesh_node.storage import SqliteStore

MAX_FIBER_BYTES: int = int(
    os.environ.get(
        'FLEETMESH_MAX_FIBER_BYTES',
        os.environ.get('FLEETMESH_ARCADE_MAX_BINARY_BYTES', str(16 * 1024 * 1024)),
    )
)
MAX_OWNER_BYTES: int = int(
    os.environ.get(
        'FLEETMESH_MAX_OWNER_FIBER_BYTES',
        os.environ.get('FLEETMESH_ARCADE_MAX_OWNER_BYTES', str(256 * 1024 * 1024)),
    )
)
MAX_UPLOAD_BODY_BYTES: int = (MAX_FIBER_BYTES * 4) // 3 + 256 * 1024

PAYLOAD_KINDS: tuple[str, ...] = (
    'video',
    'audio',
    'image',
    'text',
    'captions',
    'subtitle',
    'code',
    'binary',
    'metadata',
    'log',
)
INTERACTION_KINDS: tuple[str, ...] = ('like', 'hide', 'reaction', 'scrobble', 'view')
FIBER_KINDS: tuple[str, ...] = PAYLOAD_KINDS + INTERACTION_KINDS + ('identity',)
VISIBILITIES: tuple[str, ...] = ('public', 'vault_only')

_SAFE_BINARY_MIME: frozenset[str] = frozenset({
    'application/octet-stream',
    'application/wasm',
    'application/x-gameboy-rom',
    'application/x-gameboy-color-rom',
    'audio/mpeg',
    'audio/ogg',
    'audio/flac',
    'audio/wav',
    'video/mp4',
    'video/webm',
    'image/png',
    'image/jpeg',
    'image/webp',
    'image/gif',
    'text/vtt',
})
_ROOT_RE = re.compile(r'^[0-9a-f]{64}$')
_RANGE_HEADER_RE = re.compile(r'^bytes=(\d*)-(\d*)$')
JSON_CT = 'application/json; charset=utf-8'


[docs]class RhioBlobRecord(TypedDict): hash: str size: int tree_root: str synced_s3: bool created_at: str
[docs]class RhioStoreError(FleetmeshError): """Base error for Rhio blob storage and tree hash verification.""" pass
[docs]class FiberAccessError(Exception): """An HTTP-mappable refusal (`status`, machine-readable `error`) for fiber operations."""
[docs] def __init__(self, status: int, error: str) -> None: super().__init__(error) self.status = status self.error = error
def _norm_principal_key(raw: Any) -> str: return str(raw or '').strip().lower().lstrip('@') def _principal_key(principal: Any) -> str: if not principal: return '' if isinstance(principal, str): return _norm_principal_key(principal) if isinstance(principal, Mapping): return _norm_principal_key(principal.get('identity_id')) or _norm_principal_key(principal.get('did')) return '' def _require_principal(principal: Any) -> Mapping[str, Any]: if isinstance(principal, str): principal = {'identity_id': principal, 'did': principal if principal.startswith('did:') else '', 'handle': ''} if not isinstance(principal, Mapping) or not _principal_key(principal): raise FiberAccessError(401, 'authentication_required') return principal def _author_ref(principal: Any) -> dict[str, str]: if isinstance(principal, str): return { 'identity_id': principal, 'did': principal if principal.startswith('did:') else '', 'handle': '', } if not isinstance(principal, Mapping): return {'identity_id': '', 'did': '', 'handle': ''} return { 'identity_id': str(principal.get('identity_id') or ''), 'did': str(principal.get('did') or ''), 'handle': str(principal.get('handle') or ''), } def _safe_int(raw: Any, default: int = 0) -> int: try: return int(raw) except (TypeError, ValueError): return default def _decode_b64(raw: Any, max_bytes: int) -> Optional[bytes]: clean = str(raw or '').strip() if not clean: return None try: data = base64.b64decode(clean, validate=True) except (binascii.Error, ValueError): raise FiberAccessError(400, 'invalid_base64') if len(data) > max_bytes: raise FiberAccessError(413, 'binary_too_large') return data from fleetmesh_node.protocols.isotime import now_iso as _now_iso # noqa: E402
[docs]class RhioBaoTree: """Bao-compatible hierarchical chunked Merkle tree hashing.""" CHUNK_SIZE: int = 1024 SUBTREE_POW: int = 13 # 2^13 * 1024 B = 8 MiB exact Merkle subtree segments SEGMENT_SIZE: int = CHUNK_SIZE * (1 << SUBTREE_POW) # 8,388,608 bytes (8 MiB)
[docs] @classmethod def leaf_count(cls, size_bytes: int, chunk_size: int = CHUNK_SIZE) -> int: """Return the number of `chunk_size`-byte leaves for `size_bytes` (minimum 1).""" return _bao.bao_leaf_count(size_bytes, chunk_size=chunk_size)
[docs] @classmethod def compute_root(cls, data: bytes, chunk_size: int = CHUNK_SIZE) -> str: """Compute the Bao-compatible Merkle root hex digest.""" root, _ = cls.compute_tree(data, chunk_size=chunk_size) return root
[docs] @classmethod def compute_tree(cls, data: bytes, chunk_size: int = CHUNK_SIZE) -> tuple[str, list[bytes]]: """Compute Merkle tree root and leaf hashes for data (`fleetmesh_core.textile.bao`).""" if not data: return _bao.EMPTY_BAO_ROOT_HEX, [hashlib.sha256(b'\x00').digest()] leaves = [hashlib.sha256(b'\x00' + data[i:i + chunk_size]).digest() for i in range(0, len(data), chunk_size)] return _bao._fold(list(leaves)).hex(), leaves
[docs] @classmethod def promote_segment_root( cls, segment_root_hex: str, segment_size: int, *, is_single_segment: bool, chunk_size: int = CHUNK_SIZE, subtree_pow: int = SUBTREE_POW, ) -> str: """Promote a standalone segment Bao root to its Level-`subtree_pow` subtree root.""" return _bao.promote_segment_root( segment_root_hex, segment_size, is_single_segment=is_single_segment, chunk_size=chunk_size, subtree_pow=subtree_pow )
[docs] @classmethod def combine_segment_roots( cls, segments: Sequence[tuple[str, int]], *, chunk_size: int = CHUNK_SIZE, subtree_pow: int = SUBTREE_POW, ) -> str: """Combine `(segment_bao_root_hex, segment_byte_size)` pairs into the global Bao tree root.""" return _bao.combine_segment_roots(segments, chunk_size=chunk_size, subtree_pow=subtree_pow)
[docs] @classmethod def compute_file_root( cls, file_path: str, *, chunk_size: int = CHUNK_SIZE, subtree_pow: int = SUBTREE_POW, on_segment: Optional[Callable[[int, int, int, int], None]] = None, ) -> tuple[str, str, int]: """Stream `file_path` in `2^subtree_pow * chunk_size` (8 MiB) segments and return `(sha256_hex, tree_root_hex, total_size)`.""" seg_max = chunk_size * (1 << subtree_pow) file_size = os.path.getsize(file_path) total_segments = max(1, (file_size + seg_max - 1) // seg_max) if file_size > 0 else 0 sha = hashlib.sha256() segments: list[tuple[str, int]] = [] total_size = 0 with open(file_path, 'rb') as f: while True: buf = f.read(seg_max) if not buf: break sha.update(buf) total_size += len(buf) seg_root = cls.compute_root(buf, chunk_size=chunk_size) segments.append((seg_root, len(buf))) if on_segment is not None: on_segment(len(segments), max(len(segments), total_segments), total_size, file_size) if total_size == 0: empty_root = hashlib.sha256(b'\x00').hexdigest() return sha.hexdigest(), empty_root, 0 tree_root = cls.combine_segment_roots(segments, chunk_size=chunk_size, subtree_pow=subtree_pow) return sha.hexdigest(), tree_root, total_size
[docs] @classmethod def verify(cls, data: bytes, expected_root: str, chunk_size: int = CHUNK_SIZE) -> bool: """Verify that data produces expected Merkle tree root.""" actual_root, _ = cls.compute_tree(data, chunk_size=chunk_size) return actual_root.lower() == expected_root.lower().strip()
[docs]class RhioBlobSync(SqliteStore): """Metadata and CAS storage for Rhio cluster blobs. A `SqliteStore` rather than a hand-rolled `sqlite3.connect`: the base owns the connection, the lock, the schema version (per store, not per database - which matters the moment two stores share one server), transaction control and the dialect that rewrites `?` and `INSERT OR IGNORE` for PostgreSQL. This store opened its own connection per call and declared its schema as a SQL string, so it was SQLite-only and invisible to the locator - see docs/storage-backends.md. """ SCHEMA = ( Table('rhio_blobs', ( Column('hash', 'TEXT', primary_key=True), Column('size', 'INTEGER', not_null=True), Column('tree_root', 'TEXT', not_null=True), Column('synced_s3', 'INTEGER', not_null=True, default='0'), Column('created_at', 'TEXT', not_null=True), ), indices=( Index('idx_rhio_blobs_tree', ('tree_root',)), )), Table('rhio_fiber_claims', ( Column('fiber_root', 'TEXT', not_null=True), Column('author_key', 'TEXT', not_null=True), Column('payload_json', 'TEXT', not_null=True), Column('created_at', 'INTEGER', not_null=True), ), primary_key=('fiber_root', 'author_key'), indices=( Index('idx_rhio_fiber_claims_root', ('fiber_root',)), Index('idx_rhio_fiber_claims_author', ('author_key',)), )), ) _EXTERNAL_FILE_THRESHOLD: int = 32 * 1024 * 1024 # 32 MiB
[docs] def __init__( self, db_path: Optional[str] = ":memory:", cas_dir: Optional[str] = None, overflow_cas_dirs: Optional[Sequence[str]] = None, ): db_path = (db_path or ":memory:").strip() or ":memory:" # A file-backed store with no cas_dir kept the bytes in a process dict # while persisting the row that says it holds them. After a restart # has_blob() was true, count_blobs() counted it, synced_s3 said the # cluster already had a copy - and get_blob() returned nothing. Two of # the three callers passed a directory; the video compartment did not, # and nothing could tell, because every suite used `:memory:` where one # connection reads back whatever was written. # # So derive it rather than trust each caller to remember: a persistent # database gets a persistent CAS beside it. The dict stays for # `:memory:` alone, where losing the bytes with the rows is honest. if db_path != ":memory:": os.makedirs(os.path.dirname(os.path.abspath(db_path)), exist_ok=True) if cas_dir is None: cas_dir = os.path.splitext(os.path.abspath(db_path))[0] + '_blobs' self._cas_dir = cas_dir self._mem_blobs: dict[str, bytes] = {} self._external_files: dict[str, str] = {} self._claims: dict[str, dict[str, dict[str, Any]]] = {} self._by_target: dict[str, set[tuple[str, str]]] = {} self._root_meta: dict[str, RhioBlobRecord] = {} if cas_dir is not None: os.makedirs(cas_dir, exist_ok=True) raw_env_ov = ( os.environ.get('FLEETMESH_OVERFLOW_CAS_DIRS') or os.environ.get('RHIO_OVERFLOW_CAS_DIRS') or '' ).strip() init_ov: list[str] = [] if overflow_cas_dirs: init_ov.extend(str(d).strip() for d in overflow_cas_dirs if str(d).strip()) elif raw_env_ov: for part in raw_env_ov.replace(',', ':').split(':'): p = part.strip() if p: init_ov.append(p) self._overflow_cas_dirs: list[str] = [] # Local files evict_local_file could not delete (permissions, a # concurrent remover); the metadata row is kept either way. self.evict_errors: int = 0 self._primary_min_free_bytes: int = int( os.environ.get('FLEETMESH_PRIMARY_MIN_FREE_BYTES', str(5 * 1024 * 1024 * 1024)) ) super().__init__(db_path) if init_ov: self.set_overflow_cas_dirs(init_ov)
def _post_schema(self) -> None: for h_raw, sz_raw, root_raw, s3_raw, cat_raw in self._all( "SELECT hash, size, tree_root, synced_s3, created_at FROM rhio_blobs" ): r = str(root_raw or '').strip().lower() if r: self._root_meta[r] = { "hash": str(h_raw), "size": int(sz_raw), "tree_root": r, "synced_s3": bool(s3_raw), "created_at": str(cat_raw), } for root_raw, akey_raw, pjson in self._all( "SELECT fiber_root, author_key, payload_json FROM rhio_fiber_claims" ): try: claim = json.loads(str(pjson)) except (TypeError, ValueError): continue root = str(root_raw or '').strip().lower() akey = str(akey_raw or '').strip().lower() if isinstance(claim, dict) and root and akey: self._claims.setdefault(root, {})[akey] = claim self._index_target(root, akey, claim)
[docs] def set_overflow_cas_dirs(self, dirs: Sequence[str]) -> list[str]: """Configure secondary/overflow CAS directories (e.g. attached block storage volumes).""" cleaned: list[str] = [] primary_abs = os.path.abspath(self._cas_dir) if self._cas_dir else '' with self._lock: for raw_d in dirs: d = str(raw_d or '').strip() if not d: continue abs_d = os.path.abspath(d) if abs_d == primary_abs or abs_d in cleaned: continue try: os.makedirs(abs_d, exist_ok=True) cleaned.append(abs_d) except OSError: continue self._overflow_cas_dirs = cleaned return list(cleaned)
[docs] def get_overflow_cas_dirs(self) -> list[str]: with self._lock: return list(self._overflow_cas_dirs)
def _select_target_cas_dir(self, required_bytes: int = 0) -> Optional[str]: """Return `self._cas_dir` when it has sufficient free space, or fail over to `_overflow_cas_dirs`.""" if not self._cas_dir: return None need = max(int(required_bytes) + 64 * 1024 * 1024, self._primary_min_free_bytes) if not self._overflow_cas_dirs: return self._cas_dir try: pri_free = shutil.disk_usage(self._cas_dir).free if pri_free >= need: return self._cas_dir except OSError: pri_free = 0 best_dir = self._cas_dir best_free = pri_free for ov_dir in list(self._overflow_cas_dirs): try: os.makedirs(ov_dir, exist_ok=True) ov_free = shutil.disk_usage(ov_dir).free if ov_free >= int(required_bytes) + 64 * 1024 * 1024 and ov_free > best_free: return ov_dir if ov_free > best_free: best_free = ov_free best_dir = ov_dir except OSError: continue return best_dir
[docs] def store_blob( self, data: bytes, *, sha256_hex: Optional[str] = None, tree_root: Optional[str] = None, chunk_count: Optional[int] = None, ) -> tuple[str, str, int]: """Store blob, computing sha256 hash and Bao tree root if not provided. Returns (sha256_hash, tree_root, size). """ blob_hash = sha256_hex.strip().lower() if sha256_hex else hashlib.sha256(data).hexdigest() final_root = tree_root.strip().lower() if tree_root else RhioBaoTree.compute_root(data) size = len(data) _ = chunk_count with self._lock: # Save raw bytes if self._cas_dir: target_dir = self._select_target_cas_dir(required_bytes=size) or self._cas_dir path = os.path.join(target_dir, blob_hash) if not os.path.exists(path): try: with open(path, "wb") as f: f.write(data) except OSError: for ov_dir in self._overflow_cas_dirs: if ov_dir == target_dir: continue try: ov_path = os.path.join(ov_dir, blob_hash) with open(ov_path, "wb") as f: f.write(data) break except OSError: continue else: raise else: self._mem_blobs[blob_hash] = data # INSERT OR IGNORE preserves synced_s3 and the original created_at. # The dialect rewrites it for a backend that spells it differently. created_iso = _now_iso() with self.transaction() as cur: cur.execute( """ INSERT OR IGNORE INTO rhio_blobs (hash, size, tree_root, synced_s3, created_at) VALUES (?, ?, ?, 0, ?) """, (blob_hash, size, final_root, created_iso), ) prev_meta = self._root_meta.get(final_root) self._root_meta[final_root] = { "hash": blob_hash, "size": size, "tree_root": final_root, "synced_s3": bool(prev_meta.get("synced_s3")) if prev_meta else False, "created_at": str(prev_meta.get("created_at") or created_iso) if prev_meta else created_iso, } return blob_hash, final_root, size
[docs] def store_blob_from_file( self, file_path: str, *, sha256_hex: Optional[str] = None, tree_root: Optional[str] = None, size: Optional[int] = None, move: bool = False, chunk_count: Optional[int] = None, on_segment: Optional[Callable[[int, int, int, int], None]] = None, ) -> tuple[str, str, int]: """Store a file-backed blob into CAS without loading the full file into memory.""" _ = chunk_count if not sha256_hex or not tree_root or size is None: comp_sha, comp_root, comp_size = RhioBaoTree.compute_file_root( file_path, on_segment=on_segment ) blob_hash = (sha256_hex or comp_sha).lower().strip() final_root = (tree_root or comp_root).lower().strip() final_size = int(size if size is not None else comp_size) else: blob_hash = sha256_hex.lower().strip() final_root = tree_root.lower().strip() final_size = int(size) if self._cas_dir: with self._lock: overflow_dirs = list(self._overflow_cas_dirs) target_cas_dir = self._select_target_cas_dir(required_bytes=final_size) or self._cas_dir dest_path = os.path.join(target_cas_dir, blob_hash) src_abs = os.path.abspath(file_path) dst_abs = os.path.abspath(dest_path) if src_abs != dst_abs: need_stage = True existing_loc = self.get_blob_path(blob_hash) if existing_loc and os.path.isfile(existing_loc): try: if os.path.getsize(existing_loc) == final_size: need_stage = False except OSError: need_stage = True if need_stage: placed = False cas_parent = os.path.dirname(os.path.abspath(self._cas_dir)) alt_candidates = [ os.path.join(cas_parent, 'uploads', os.path.basename(file_path)), os.path.join(cas_parent, 'uploads', 'staging', os.path.basename(file_path)), ] if move: for cand_src in (file_path, *alt_candidates): if not os.path.isfile(cand_src): continue try: os.rename(cand_src, dest_path) placed = True if cand_src != file_path and os.path.exists(file_path): try: os.unlink(file_path) except OSError: pass break except OSError: placed = False else: for cand_src in (file_path, *alt_candidates): if not os.path.isfile(cand_src): continue for link_dir in (target_cas_dir, self._cas_dir, *overflow_dirs): if not link_dir: continue cand_dest = os.path.join(link_dir, blob_hash) try: if os.path.getsize(cand_src) != final_size: continue if os.path.exists(cand_dest): os.unlink(cand_dest) os.link(cand_src, cand_dest) dest_path = cand_dest placed = True break except OSError: placed = False if placed: break if not placed: candidate_dirs = [target_cas_dir] + [ d for d in (self._cas_dir, *overflow_dirs) if d and d != target_cas_dir ] last_err: Optional[OSError] = None for write_dir in candidate_dirs: cand_dest = os.path.join(write_dir, blob_hash) tmp_path = f"{cand_dest}.tmp.{os.getpid()}.{threading.get_ident()}" try: shutil.copyfile(file_path, tmp_path) os.replace(tmp_path, cand_dest) dest_path = cand_dest placed = True break except OSError as exc: last_err = exc finally: if os.path.exists(tmp_path): try: os.unlink(tmp_path) except OSError: pass if not placed: if not move and os.path.isfile(file_path): with self._lock: self._external_files[blob_hash] = os.path.abspath(file_path) placed = True elif last_err is not None: raise last_err if move and os.path.exists(file_path): try: os.unlink(file_path) except OSError: pass elif move and os.path.exists(file_path): try: os.unlink(file_path) except OSError: pass with self._lock: if not self._cas_dir: if final_size > self._EXTERNAL_FILE_THRESHOLD: self._external_files[blob_hash] = os.path.abspath(file_path) else: with open(file_path, "rb") as f: self._mem_blobs[blob_hash] = f.read() if move and os.path.exists(file_path): try: os.unlink(file_path) except OSError: pass created_iso = _now_iso() with self.transaction() as cur: cur.execute( """ INSERT OR IGNORE INTO rhio_blobs (hash, size, tree_root, synced_s3, created_at) VALUES (?, ?, ?, 0, ?) """, (blob_hash, final_size, final_root, created_iso), ) prev_meta = self._root_meta.get(final_root) self._root_meta[final_root] = { "hash": blob_hash, "size": final_size, "tree_root": final_root, "synced_s3": bool(prev_meta.get("synced_s3")) if prev_meta else False, "created_at": str(prev_meta.get("created_at") or created_iso) if prev_meta else created_iso, } return blob_hash, final_root, final_size
[docs] def get_blob_path(self, blob_hash: str) -> Optional[str]: """Return the filesystem path of a stored CAS blob if backed by `cas_dir`, `_overflow_cas_dirs`, or an external file.""" clean_hash = blob_hash.lower().strip() if not clean_hash: return None with self._lock: if self._cas_dir: path = os.path.join(self._cas_dir, clean_hash) if os.path.isfile(path): return path for ov_dir in self._overflow_cas_dirs: ov_path = os.path.join(ov_dir, clean_hash) if os.path.isfile(ov_path): return ov_path ext_path = self._external_files.get(clean_hash) if ext_path and os.path.isfile(ext_path): return ext_path return None
[docs] def has_local_blob(self, blob_hash: str) -> bool: """Return True if the raw bytes for `blob_hash` are currently present on local/overflow disk or in memory.""" clean_hash = blob_hash.lower().strip() if not clean_hash: return False if self.get_blob_path(clean_hash) is not None: return True with self._lock: return clean_hash in self._mem_blobs
[docs] def migrate_blob_to_overflow( self, blob_hash: str, target_dir: Optional[str] = None, ) -> Optional[str]: """Move `blob_hash` from primary `_cas_dir` to an overflow CAS volume and return the new path.""" clean_hash = blob_hash.lower().strip() if not clean_hash or not self._cas_dir: return None pri_path = os.path.join(self._cas_dir, clean_hash) if not os.path.isfile(pri_path): return self.get_blob_path(clean_hash) try: fsize = os.path.getsize(pri_path) except OSError: return None candidates = [target_dir] if target_dir else list(self._overflow_cas_dirs) for ov_dir in candidates: if not ov_dir: continue abs_ov = os.path.abspath(ov_dir) if abs_ov == os.path.abspath(self._cas_dir): continue try: os.makedirs(abs_ov, exist_ok=True) if shutil.disk_usage(abs_ov).free < fsize + 32 * 1024 * 1024: continue dest_path = os.path.join(abs_ov, clean_hash) if not os.path.isfile(dest_path) or os.path.getsize(dest_path) != fsize: tmp_path = f"{dest_path}.mig.{os.getpid()}.{threading.get_ident()}" try: shutil.copyfile(pri_path, tmp_path) os.replace(tmp_path, dest_path) finally: if os.path.exists(tmp_path): try: os.unlink(tmp_path) except OSError: pass if os.path.isfile(dest_path) and os.path.getsize(dest_path) == fsize: with self._lock: if abs_ov not in self._overflow_cas_dirs: self._overflow_cas_dirs.append(abs_ov) try: os.unlink(pri_path) except OSError: pass return dest_path except OSError: continue return None
def _remove_file(self, path: str) -> int: """Delete `path` and return the bytes freed; a failure is counted in evict_errors.""" try: size = os.path.getsize(path) os.unlink(path) except OSError: self.evict_errors += 1 return 0 return size
[docs] def evict_local_file(self, blob_hash: str, *, include_overflow: bool = False) -> int: """Evict the local file for `blob_hash` to free disk space while preserving its `rhio_blobs` metadata row.""" clean_hash = blob_hash.lower().strip() if not clean_hash: return 0 freed_bytes = 0 with self._lock: dirs_to_check = [self._cas_dir] if self._cas_dir else [] if include_overflow: dirs_to_check.extend(self._overflow_cas_dirs) for d in dirs_to_check: if not d: continue path = os.path.join(d, clean_hash) if os.path.isfile(path): freed_bytes += self._remove_file(path) if clean_hash in self._mem_blobs: freed_bytes += len(self._mem_blobs.pop(clean_hash)) ext_path = self._external_files.pop(clean_hash, None) if ext_path and os.path.isfile(ext_path): freed_bytes += self._remove_file(ext_path) return freed_bytes
[docs] def get_blob(self, blob_hash: str) -> Optional[bytes]: """Retrieve blob bytes by sha256 hash.""" clean_hash = blob_hash.lower().strip() path = self.get_blob_path(clean_hash) if path and os.path.isfile(path): with open(path, "rb") as f: return f.read() with self._lock: if clean_hash in self._mem_blobs: return self._mem_blobs[clean_hash] return None
[docs] def get_blob_meta(self, blob_hash: str) -> Optional[RhioBlobRecord]: """Retrieve stored metadata (hash, size, tree_root, synced_s3, created_at) without reading the blob.""" clean_hash = blob_hash.lower().strip() if not clean_hash: return None with self._lock: row = self._one( "SELECT hash, size, tree_root, synced_s3, created_at FROM rhio_blobs WHERE hash = ?", (clean_hash,), ) if not row: return None return { "hash": str(row[0]), "size": int(row[1]), "tree_root": str(row[2]), "synced_s3": bool(row[3]), "created_at": str(row[4]), }
[docs] def get_blob_slice(self, blob_hash: str, start: int, length: int) -> Optional[bytes]: """Read a byte range `[start : start + length]` directly from CAS disk without loading the full blob into RAM.""" clean_hash = blob_hash.lower().strip() if start < 0 or length < 0: return None path = self.get_blob_path(clean_hash) if path and os.path.isfile(path): with open(path, "rb") as f: f.seek(start) return f.read(length) with self._lock: raw = self._mem_blobs.get(clean_hash) if raw is None: return None return raw[start : start + length]
[docs] def get_blob_meta_by_root(self, tree_root: str) -> Optional[RhioBlobRecord]: """Retrieve stored blob metadata by its 1,024-byte-leaf Bao `tree_root`.""" clean_root = str(tree_root or '').lower().strip() if not clean_root: return None with self._lock: cached = self._root_meta.get(clean_root) if cached is not None: return dict(cached) # type: ignore[return-value] row = self._one( "SELECT hash, size, tree_root, synced_s3, created_at FROM rhio_blobs WHERE tree_root = ?", (clean_root,), ) if not row: return None rec: RhioBlobRecord = { "hash": str(row[0]), "size": int(row[1]), "tree_root": str(row[2]), "synced_s3": bool(row[3]), "created_at": str(row[4]), } self._root_meta[clean_root] = rec return dict(rec) # type: ignore[return-value]
[docs] def get_blob_path_by_root(self, tree_root: str) -> Optional[str]: """Return the CAS filesystem path for the blob matching `tree_root`.""" meta = self.get_blob_meta_by_root(tree_root) return self.get_blob_path(meta["hash"]) if meta else None
[docs] def get_blob_by_root(self, tree_root: str) -> Optional[bytes]: """Retrieve blob bytes by `tree_root`.""" meta = self.get_blob_meta_by_root(tree_root) return self.get_blob(meta["hash"]) if meta else None
[docs] def get_blob_slice_by_root(self, tree_root: str, start: int, length: int) -> Optional[bytes]: """Read a byte range `[start : start + length]` by `tree_root` without loading the full blob into RAM.""" meta = self.get_blob_meta_by_root(tree_root) return self.get_blob_slice(meta["hash"], start, length) if meta else None
[docs] def delete_blob(self, blob_hash: str) -> bool: """Delete an ephemeral blob from CAS disk/memory and metadata store.""" clean_hash = blob_hash.lower().strip() if not clean_hash: return False removed = False with self._lock: meta = self.get_blob_meta(clean_hash) root_to_check = str(meta["tree_root"]).lower() if meta else "" for d in ([self._cas_dir] if self._cas_dir else []) + list(self._overflow_cas_dirs): if not d: continue path = os.path.join(d, clean_hash) if os.path.exists(path): try: os.unlink(path) except OSError: pass else: removed = True if clean_hash in self._mem_blobs: del self._mem_blobs[clean_hash] removed = True if clean_hash in self._external_files: del self._external_files[clean_hash] removed = True with self.transaction() as cur: cur.execute("DELETE FROM rhio_blobs WHERE hash = ?", (clean_hash,)) stale_roots = [ r for r, rm in self._root_meta.items() if rm.get("hash") == clean_hash ] if root_to_check and root_to_check not in stale_roots: stale_roots.append(root_to_check) for r_del in stale_roots: self._root_meta.pop(r_del, None) if self._one("SELECT 1 FROM rhio_blobs WHERE tree_root = ?", (r_del,)) is None: old_claims = self._claims.pop(r_del, {}) for akey, clm in old_claims.items(): self._unindex_target(r_del, akey, clm) cur.execute("DELETE FROM rhio_fiber_claims WHERE fiber_root = ?", (r_del,)) return removed
[docs] def has_blob(self, blob_hash: str) -> bool: """Check if blob exists in storage.""" clean_hash = blob_hash.lower().strip() with self._lock: if clean_hash in self._external_files and os.path.isfile(self._external_files[clean_hash]): return True return self._one("SELECT 1 FROM rhio_blobs WHERE hash = ?", (clean_hash,)) is not None
[docs] def mark_s3_synced(self, blob_hash: str) -> None: """Mark blob as synchronized with S3 / MinIO cluster storage.""" clean_hash = blob_hash.lower().strip() with self._lock: with self.transaction() as cur: cur.execute("UPDATE rhio_blobs SET synced_s3 = 1 WHERE hash = ?", (clean_hash,)) for rmeta in self._root_meta.values(): if rmeta.get("hash") == clean_hash: rmeta["synced_s3"] = True
[docs] def count_blobs(self) -> int: with self._lock: row = self._one("SELECT COUNT(*) FROM rhio_blobs") return int(row[0]) if row else 0
[docs] def list_blobs(self, limit: int = 100) -> list[RhioBlobRecord]: with self._lock: rows = self._all( "SELECT hash, size, tree_root, synced_s3, created_at FROM rhio_blobs " "ORDER BY created_at DESC LIMIT ?", (limit,), ) return [ { "hash": str(r[0]), "size": int(r[1]), "tree_root": str(r[2]), "synced_s3": bool(r[3]), "created_at": str(r[4]), } for r in rows ]
# ---- Unified Content-Addressed Fiber Claims ----------------------------- def _index_target(self, root: str, key: str, claim: Mapping[str, Any]) -> None: target = str((claim.get('target') or {}).get('strand_id') or '') if target: self._by_target.setdefault(target, set()).add((root, key)) def _unindex_target(self, root: str, key: str, claim: Mapping[str, Any]) -> None: target = str((claim.get('target') or {}).get('strand_id') or '') refs = self._by_target.get(target) if refs is not None: refs.discard((root, key)) if not refs: del self._by_target[target] def _viewer_claim(self, root: str, principal: Optional[Mapping[str, Any]]) -> Optional[dict[str, Any]]: """The caller's own claim, else the earliest public one; `None` means not readable.""" claims = self._claims.get(root) or {} own = claims.get(_principal_key(principal)) if principal else None if own is not None: return own public = [c for c in claims.values() if c.get('visibility') == 'public'] return min(public, key=lambda c: _safe_int(c.get('created_at'))) if public else None def _manifest( self, root: str, claim: Mapping[str, Any], meta: Optional[RhioBlobRecord] = None, ) -> dict[str, Any]: if meta is None: meta = self.get_blob_meta_by_root(root) size = int(meta["size"]) if meta else 0 sha = str(meta["hash"]) if meta else "" out: dict[str, Any] = { 'fiber_root': root, 'sha256_hex': sha, 'byte_length': size, 'leaf_count_1024b': RhioBaoTree.leaf_count(size), 'kind': claim.get('kind'), 'author': dict(claim.get('author') or {}), 'visibility': claim.get('visibility'), 'mime_type': claim.get('mime_type') or '', 'created_at': claim.get('created_at'), } if claim.get('target'): out['target'] = dict(claim['target']) return out
[docs] def has_fiber(self, root: str) -> bool: clean_root = str(root or '').strip().lower() if not clean_root: return False with self._lock: return bool(self._claims.get(clean_root)) and self.get_blob_meta_by_root(clean_root) is not None
[docs] def readable(self, root: str, principal: Optional[Mapping[str, Any]] = None) -> bool: return self.manifest(root, principal) is not None
[docs] def manifest(self, root: str, principal: Optional[Mapping[str, Any]] = None) -> Optional[dict[str, Any]]: clean_root = str(root or '').strip().lower() if not clean_root: return None with self._lock: meta = self.get_blob_meta_by_root(clean_root) if meta is None: return None claim = self._viewer_claim(clean_root, principal) return self._manifest(clean_root, claim, meta) if claim else None
[docs] def get(self, root: str, principal: Optional[Mapping[str, Any]] = None) -> Optional[tuple[bytes, dict[str, Any]]]: clean_root = str(root or '').strip().lower() if not clean_root: return None with self._lock: m = self.manifest(clean_root, principal) if m is None: return None raw = self.get_blob(str(m.get('sha256_hex') or '')) return (raw, m) if raw is not None else None
[docs] def list_own(self, principal: Optional[Mapping[str, Any]]) -> list[dict[str, Any]]: key = _principal_key(principal) if not key: return [] with self._lock: out: list[dict[str, Any]] = [] for root, claims in self._claims.items(): if key in claims: meta = self.get_blob_meta_by_root(root) if meta is not None: out.append(self._manifest(root, claims[key], meta)) return sorted(out, key=lambda m: _safe_int(m.get('created_at')), reverse=True)
[docs] def owner_bytes(self, principal: Optional[Mapping[str, Any]]) -> int: key = _principal_key(principal) if not key: return 0 with self._lock: total = 0 for root, claims in self._claims.items(): if key in claims: meta = self.get_blob_meta_by_root(root) if meta is not None: total += int(meta["size"]) return total
[docs] def interaction_counts(self, target_strand_id: str) -> dict[str, int]: """Public interaction claims on a strand, by kind (each actor's like counts once).""" counts: dict[str, int] = {} with self._lock: for root, key in self._by_target.get(target_strand_id, ()): claim = (self._claims.get(root) or {}).get(key) or {} kind = str(claim.get('kind') or '') if claim.get('visibility') == 'public' and kind in INTERACTION_KINDS: counts[kind] = counts.get(kind, 0) + 1 return counts
[docs] def claim_blob( self, tree_root: str, principal: Optional[Mapping[str, Any]], *, sha256_hex: str = '', size: Optional[int] = None, kind: str = 'binary', mime_type: str = '', visibility: str = 'public', target_strand_id: str = '', overwrite_visibility: bool = True, created_at: Optional[int] = None, ) -> Optional[dict[str, Any]]: """Register or update an author's fiber claim on an existing (or metadata-registered) blob.""" clean_root = str(tree_root or '').strip().lower() if len(clean_root) != 64: return None who = principal if (principal and _principal_key(principal)) else {'identity_id': 'node', 'did': '', 'handle': ''} key = _principal_key(who) or 'node' vis = 'vault_only' if visibility == 'vault_only' else 'public' eff_kind = kind if kind in FIBER_KINDS else 'binary' now = int(created_at if created_at is not None else time.time()) with self._lock: meta = self.get_blob_meta_by_root(clean_root) if meta is None and sha256_hex and size is not None: clean_sha = str(sha256_hex).strip().lower() if len(clean_sha) == 64: with self.transaction() as cur: cur.execute( """ INSERT OR IGNORE INTO rhio_blobs (hash, size, tree_root, synced_s3, created_at) VALUES (?, ?, ?, 0, ?) """, (clean_sha, int(size), clean_root, _now_iso()), ) meta = self.get_blob_meta_by_root(clean_root) if meta is None: return None claims = self._claims.setdefault(clean_root, {}) existing = claims.get(key) claim = dict(existing or {'author': _author_ref(who), 'kind': eff_kind, 'created_at': now}) if existing is None or overwrite_visibility: claim['visibility'] = vis if mime_type or not claim.get('mime_type'): claim['mime_type'] = str(mime_type or claim.get('mime_type') or '')[:120] if kind in FIBER_KINDS: claim['kind'] = kind if target_strand_id: if existing is not None: self._unindex_target(clean_root, key, existing) claim['target'] = {'strand_id': str(target_strand_id)[:200]} claims[key] = claim self._index_target(clean_root, key, claim) pjson = json.dumps(claim, separators=(',', ':')) with self.transaction() as cur: cur.execute( """ INSERT OR REPLACE INTO rhio_fiber_claims (fiber_root, author_key, payload_json, created_at) VALUES (?, ?, ?, ?) """, (clean_root, key, pjson, _safe_int(claim.get('created_at'), now)), ) return self._manifest(clean_root, claim, meta)
[docs] def put( self, raw: bytes, principal: Optional[Mapping[str, Any]], *, kind: str, mime_type: str = '', visibility: str = 'vault_only', target_strand_id: str = '', enforce_quota: bool = True, ) -> dict[str, Any]: """Store `raw` (once) in `RhioBlobSync` and record the caller's claim on it; returns the caller's manifest.""" who = _require_principal(principal) if kind not in FIBER_KINDS: raise FiberAccessError(400, 'invalid_fiber_kind') if kind in INTERACTION_KINDS and not target_strand_id: raise FiberAccessError(400, 'interaction_target_required') if len(raw) > MAX_FIBER_BYTES: raise FiberAccessError(413, 'binary_too_large') vis = visibility if visibility in VISIBILITIES else 'vault_only' root = RhioBaoTree.compute_root(raw) key = _principal_key(who) now = int(time.time()) with self._lock: existing = (self._claims.get(root) or {}).get(key) if existing is None and enforce_quota and self.owner_bytes(who) + len(raw) > MAX_OWNER_BYTES: raise FiberAccessError(413, 'owner_quota_exceeded') sha, _, size = self.store_blob(raw, tree_root=root) manifest = self.claim_blob( root, who, sha256_hex=sha, size=size, kind=kind, mime_type=mime_type, visibility=vis, target_strand_id=target_strand_id, overwrite_visibility=True, created_at=now, ) assert manifest is not None return manifest
[docs] def set_visibility(self, root: str, principal: Optional[Mapping[str, Any]], visibility: str) -> dict[str, Any]: who = _require_principal(principal) key = _principal_key(who) clean_root = str(root or '').strip().lower() if visibility not in VISIBILITIES: raise FiberAccessError(400, 'invalid_visibility') with self._lock: claim = (self._claims.get(clean_root) or {}).get(key) meta = self.get_blob_meta_by_root(clean_root) if claim is not None else None if claim is None or meta is None: raise FiberAccessError(404, 'fiber_claim_not_found') if claim.get('visibility') != visibility: claim['visibility'] = visibility pjson = json.dumps(claim, separators=(',', ':')) with self.transaction() as cur: cur.execute( "UPDATE rhio_fiber_claims SET payload_json = ? WHERE fiber_root = ? AND author_key = ?", (pjson, clean_root, key), ) return self._manifest(clean_root, claim, meta)
[docs] def release(self, root: str, principal: Optional[Mapping[str, Any]]) -> bool: """Drop the caller's claim; the underlying blob is deleted when no claims remain.""" who = _require_principal(principal) key = _principal_key(who) clean_root = str(root or '').strip().lower() with self._lock: claims = self._claims.get(clean_root) or {} if key not in claims: return False self._unindex_target(clean_root, key, claims.pop(key)) with self.transaction() as cur: cur.execute( "DELETE FROM rhio_fiber_claims WHERE fiber_root = ? AND author_key = ?", (clean_root, key), ) if not claims: self._claims.pop(clean_root, None) meta = self.get_blob_meta_by_root(clean_root) if meta is not None: self.delete_blob(meta["hash"]) return True
[docs] def close(self) -> None: super().close()
[docs]def binary_http_response( raw: bytes, *, bao_root: str, sha256_hex: str, mime_type: str, public: bool, filename: str, as_json: bool, json_extra: Optional[Mapping[str, Any]] = None, ) -> tuple[int, Any, str, dict[str, str]]: """Serve stored bytes so they can never render as an active page on the app's origin.""" cache = 'public, max-age=3600, immutable' if public else 'private, no-store' hdrs: dict[str, str] = { 'X-Bao-Tree-Root': bao_root, 'Cache-Control': cache, 'X-Content-Type-Options': 'nosniff', 'Accept-Ranges': 'bytes', } mime = str(mime_type or '').strip().lower() if as_json: return 200, { 'ok': True, **dict(json_extra or {}), 'bao_root': bao_root, 'sha256_hex': sha256_hex, 'mime_type': mime or 'application/octet-stream', 'size_bytes': len(raw), 'data_b64': base64.b64encode(raw).decode('ascii'), }, JSON_CT, hdrs fname = re.sub(r'[^A-Za-z0-9._-]', '_', filename or 'fiber.bin')[:120] hdrs['Content-Disposition'] = f'attachment; filename="{fname}"' hdrs['Content-Security-Policy'] = "default-src 'none'; sandbox" return 200, raw, (mime if mime in _SAFE_BINARY_MIME else 'application/octet-stream'), hdrs
[docs]def is_fiber_route(route: str) -> bool: return route == '/api/fibers' or route.startswith('/api/fibers/')
def _not_found(root: str) -> tuple[int, Any, str, dict[str, str]]: return 404, {'ok': False, 'error': 'fiber_not_found', 'fiber_root': root}, JSON_CT, {}
[docs]def handle_fiber_http( store: Any, method: str, route: str, query: Any = '', body: Optional[Mapping[str, Any]] = None, principal: Optional[Mapping[str, Any]] = None, *, range_header: str = '', ) -> Optional[tuple[int, Any, str, dict[str, str]]]: """Route `/api/fibers*`. `principal` must come from the server's own session resolution.""" store = getattr(store, 'fiber_store', getattr(store, 'rhio_store', store)) qp = query if isinstance(query, Mapping) else urllib.parse.parse_qs(str(query or '')) try: if method in ('GET', 'HEAD'): if route == '/api/fibers': if not principal or not _principal_key(principal): raise FiberAccessError(401, 'authentication_required') return 200, {'ok': True, 'fibers': store.list_own(principal)}, JSON_CT, {'Cache-Control': 'private, no-store'} if route == '/api/fibers/interactions': target = (qp.get('target', [''])[0] or '').strip() if not target: raise FiberAccessError(400, 'target_required') return 200, {'ok': True, 'target': target, 'counts': store.interaction_counts(target)}, JSON_CT, {'Cache-Control': 'public, max-age=15'} parts = [p for p in route[len('/api/fibers/'):].split('/') if p] if not parts or len(parts) > 2 or not _ROOT_RE.match(parts[0]): return None root = parts[0] if len(parts) == 2: if parts[1] != 'manifest': return None manifest = store.manifest(root, principal) return (200, {'ok': True, 'fiber': manifest}, JSON_CT, {'Cache-Control': 'private, no-store'}) if manifest else _not_found(root) as_json = (qp.get('format', [''])[0] or '').strip().lower() == 'json' rng = str(range_header or '').strip() if method == 'HEAD' and not rng and not as_json: manifest = store.manifest(root, principal) if manifest is None: return _not_found(root) _st, _body, ct, hdrs = binary_http_response( b'', bao_root=root, sha256_hex=manifest['sha256_hex'], mime_type=manifest.get('mime_type') or '', public=manifest.get('visibility') == 'public', filename=f"{root[:16]}.bin", as_json=False, ) hdrs['Content-Length'] = str(int(manifest.get('byte_length') or 0)) return 200, b'', ct, hdrs if rng and not as_json: manifest = store.manifest(root, principal) if manifest is None: return _not_found(root) total = int(manifest.get('byte_length') or 0) m = _RANGE_HEADER_RE.match(rng) if not m: return 416, {'ok': False, 'error': 'range_not_satisfiable'}, JSON_CT, {'Content-Range': f'bytes */{total}', 'Accept-Ranges': 'bytes'} start_s, end_s = m.group(1), m.group(2) if start_s == '' and end_s == '': return 416, {'ok': False, 'error': 'range_not_satisfiable'}, JSON_CT, {'Content-Range': f'bytes */{total}', 'Accept-Ranges': 'bytes'} if start_s == '' and end_s != '': suffix_len = int(end_s) start = max(0, total - suffix_len) end = total - 1 else: start = int(start_s) if start_s else 0 end = int(end_s) if end_s else (total - 1) if total <= 0 or start >= total or start > end: return 416, {'ok': False, 'error': 'range_not_satisfiable'}, JSON_CT, {'Content-Range': f'bytes */{total}', 'Accept-Ranges': 'bytes'} end = min(end, total - 1) slice_len = end - start + 1 if method == 'HEAD': _st, _body, ct, hdrs = binary_http_response( b'', bao_root=root, sha256_hex=manifest['sha256_hex'], mime_type=manifest.get('mime_type') or '', public=manifest.get('visibility') == 'public', filename=f"{root[:16]}.bin", as_json=False, ) hdrs['Content-Range'] = f'bytes {start}-{end}/{total}' hdrs['Content-Length'] = str(slice_len) hdrs['X-Bao-Range-Root'] = root return 206, b'', ct, hdrs if hasattr(store, 'get_blob_slice') and manifest.get('sha256_hex'): chunk = store.get_blob_slice(str(manifest['sha256_hex']), start, slice_len) elif hasattr(store, 'get_blob_slice_by_root'): chunk = store.get_blob_slice_by_root(root, start, slice_len) else: found_full = store.get(root, principal) chunk = found_full[0][start : end + 1] if found_full else None if chunk is None: return _not_found(root) _st, body_b, ct, hdrs = binary_http_response( chunk, bao_root=root, sha256_hex=manifest['sha256_hex'], mime_type=manifest.get('mime_type') or '', public=manifest.get('visibility') == 'public', filename=f"{root[:16]}.bin", as_json=False, ) hdrs['Content-Range'] = f'bytes {start}-{end}/{total}' if len(chunk) <= 1024 * 1024: hdrs['X-Bao-Range-Root'] = RhioBaoTree.compute_root(chunk) else: hdrs['X-Bao-Range-Root'] = root return 206, body_b, ct, hdrs found = store.get(root, principal) if found is None: return _not_found(root) raw, manifest = found return binary_http_response( raw, bao_root=root, sha256_hex=manifest['sha256_hex'], mime_type=manifest.get('mime_type') or '', public=manifest.get('visibility') == 'public', filename=f"{root[:16]}.bin", as_json=as_json, json_extra={'fiber': manifest}, ) if method != 'POST' or not is_fiber_route(route): return None payload: Mapping[str, Any] = body or {} if route in ('/api/fibers', '/api/fibers/put'): raw = _decode_b64(payload.get('data_b64'), MAX_FIBER_BYTES) if raw is None: raise FiberAccessError(400, 'data_b64_required') manifest = store.put( raw, principal, kind=str(payload.get('kind') or ''), mime_type=str(payload.get('mime_type') or ''), visibility=str(payload.get('visibility') or 'vault_only'), target_strand_id=str(payload.get('target_strand_id') or ''), ) return 200, {'ok': True, 'fiber': manifest}, JSON_CT, {} if route == '/api/fibers/visibility': manifest = store.set_visibility( str(payload.get('fiber_root') or ''), principal, str(payload.get('visibility') or ''), ) return 200, {'ok': True, 'fiber': manifest}, JSON_CT, {} if route == '/api/fibers/release': root = str(payload.get('fiber_root') or '') released = store.release(root, principal) return (200 if released else 404), {'ok': released, 'fiber_root': root}, JSON_CT, {} return None except Exception as exc: status = getattr(exc, 'status', None) error = getattr(exc, 'error', None) if isinstance(status, int) and isinstance(error, str): return status, {'ok': False, 'error': error}, JSON_CT, {} raise