"""Rhio blob storage and Bao-compatible chunked tree hashing.
Standard library only. Provides:
- Bao chunked Merkle tree verification (1024-byte chunks).
- Local CAS blob repository and SQLite synchronization metadata.
- S3 / MinIO synchronization state tracking.
"""
from __future__ import annotations
import base64
import binascii
import hashlib
import json
import os
import re
import shutil
import threading
import time
import urllib.parse
from typing import Any, Callable, Mapping, Optional, Sequence, TypedDict
from fleetmesh_core.conformance.errors import FleetmeshError
from fleetmesh_core.textile import bao as _bao
__all__ = [
'FIBER_KINDS',
'FiberAccessError',
'INTERACTION_KINDS',
'MAX_FIBER_BYTES',
'MAX_OWNER_BYTES',
'MAX_UPLOAD_BODY_BYTES',
'PAYLOAD_KINDS',
'RhioBaoTree',
'RhioBlobRecord',
'RhioBlobSync',
'RhioStoreError',
'VISIBILITIES',
'binary_http_response',
'handle_fiber_http',
'is_fiber_route',
]
from fleetmesh_node.compartment.schema import Column, Index, Table
from fleetmesh_node.storage import SqliteStore
MAX_FIBER_BYTES: int = int(
os.environ.get(
'FLEETMESH_MAX_FIBER_BYTES',
os.environ.get('FLEETMESH_ARCADE_MAX_BINARY_BYTES', str(16 * 1024 * 1024)),
)
)
MAX_OWNER_BYTES: int = int(
os.environ.get(
'FLEETMESH_MAX_OWNER_FIBER_BYTES',
os.environ.get('FLEETMESH_ARCADE_MAX_OWNER_BYTES', str(256 * 1024 * 1024)),
)
)
MAX_UPLOAD_BODY_BYTES: int = (MAX_FIBER_BYTES * 4) // 3 + 256 * 1024
PAYLOAD_KINDS: tuple[str, ...] = (
'video',
'audio',
'image',
'text',
'captions',
'subtitle',
'code',
'binary',
'metadata',
'log',
)
INTERACTION_KINDS: tuple[str, ...] = ('like', 'hide', 'reaction', 'scrobble', 'view')
FIBER_KINDS: tuple[str, ...] = PAYLOAD_KINDS + INTERACTION_KINDS + ('identity',)
VISIBILITIES: tuple[str, ...] = ('public', 'vault_only')
_SAFE_BINARY_MIME: frozenset[str] = frozenset({
'application/octet-stream',
'application/wasm',
'application/x-gameboy-rom',
'application/x-gameboy-color-rom',
'audio/mpeg',
'audio/ogg',
'audio/flac',
'audio/wav',
'video/mp4',
'video/webm',
'image/png',
'image/jpeg',
'image/webp',
'image/gif',
'text/vtt',
})
_ROOT_RE = re.compile(r'^[0-9a-f]{64}$')
_RANGE_HEADER_RE = re.compile(r'^bytes=(\d*)-(\d*)$')
JSON_CT = 'application/json; charset=utf-8'
[docs]class RhioBlobRecord(TypedDict):
hash: str
size: int
tree_root: str
synced_s3: bool
created_at: str
[docs]class RhioStoreError(FleetmeshError):
"""Base error for Rhio blob storage and tree hash verification."""
pass
[docs]class FiberAccessError(Exception):
"""An HTTP-mappable refusal (`status`, machine-readable `error`) for fiber operations."""
[docs] def __init__(self, status: int, error: str) -> None:
super().__init__(error)
self.status = status
self.error = error
def _norm_principal_key(raw: Any) -> str:
return str(raw or '').strip().lower().lstrip('@')
def _principal_key(principal: Any) -> str:
if not principal:
return ''
if isinstance(principal, str):
return _norm_principal_key(principal)
if isinstance(principal, Mapping):
return _norm_principal_key(principal.get('identity_id')) or _norm_principal_key(principal.get('did'))
return ''
def _require_principal(principal: Any) -> Mapping[str, Any]:
if isinstance(principal, str):
principal = {'identity_id': principal, 'did': principal if principal.startswith('did:') else '', 'handle': ''}
if not isinstance(principal, Mapping) or not _principal_key(principal):
raise FiberAccessError(401, 'authentication_required')
return principal
def _author_ref(principal: Any) -> dict[str, str]:
if isinstance(principal, str):
return {
'identity_id': principal,
'did': principal if principal.startswith('did:') else '',
'handle': '',
}
if not isinstance(principal, Mapping):
return {'identity_id': '', 'did': '', 'handle': ''}
return {
'identity_id': str(principal.get('identity_id') or ''),
'did': str(principal.get('did') or ''),
'handle': str(principal.get('handle') or ''),
}
def _safe_int(raw: Any, default: int = 0) -> int:
try:
return int(raw)
except (TypeError, ValueError):
return default
def _decode_b64(raw: Any, max_bytes: int) -> Optional[bytes]:
clean = str(raw or '').strip()
if not clean:
return None
try:
data = base64.b64decode(clean, validate=True)
except (binascii.Error, ValueError):
raise FiberAccessError(400, 'invalid_base64')
if len(data) > max_bytes:
raise FiberAccessError(413, 'binary_too_large')
return data
from fleetmesh_node.protocols.isotime import now_iso as _now_iso # noqa: E402
[docs]class RhioBaoTree:
"""Bao-compatible hierarchical chunked Merkle tree hashing."""
CHUNK_SIZE: int = 1024
SUBTREE_POW: int = 13 # 2^13 * 1024 B = 8 MiB exact Merkle subtree segments
SEGMENT_SIZE: int = CHUNK_SIZE * (1 << SUBTREE_POW) # 8,388,608 bytes (8 MiB)
[docs] @classmethod
def leaf_count(cls, size_bytes: int, chunk_size: int = CHUNK_SIZE) -> int:
"""Return the number of `chunk_size`-byte leaves for `size_bytes` (minimum 1)."""
return _bao.bao_leaf_count(size_bytes, chunk_size=chunk_size)
[docs] @classmethod
def compute_root(cls, data: bytes, chunk_size: int = CHUNK_SIZE) -> str:
"""Compute the Bao-compatible Merkle root hex digest."""
root, _ = cls.compute_tree(data, chunk_size=chunk_size)
return root
[docs] @classmethod
def compute_tree(cls, data: bytes, chunk_size: int = CHUNK_SIZE) -> tuple[str, list[bytes]]:
"""Compute Merkle tree root and leaf hashes for data (`fleetmesh_core.textile.bao`)."""
if not data:
return _bao.EMPTY_BAO_ROOT_HEX, [hashlib.sha256(b'\x00').digest()]
leaves = [hashlib.sha256(b'\x00' + data[i:i + chunk_size]).digest() for i in range(0, len(data), chunk_size)]
return _bao._fold(list(leaves)).hex(), leaves
[docs] @classmethod
def combine_segment_roots(
cls,
segments: Sequence[tuple[str, int]],
*,
chunk_size: int = CHUNK_SIZE,
subtree_pow: int = SUBTREE_POW,
) -> str:
"""Combine `(segment_bao_root_hex, segment_byte_size)` pairs into the global Bao tree root."""
return _bao.combine_segment_roots(segments, chunk_size=chunk_size, subtree_pow=subtree_pow)
[docs] @classmethod
def compute_file_root(
cls,
file_path: str,
*,
chunk_size: int = CHUNK_SIZE,
subtree_pow: int = SUBTREE_POW,
on_segment: Optional[Callable[[int, int, int, int], None]] = None,
) -> tuple[str, str, int]:
"""Stream `file_path` in `2^subtree_pow * chunk_size` (8 MiB) segments and return `(sha256_hex, tree_root_hex, total_size)`."""
seg_max = chunk_size * (1 << subtree_pow)
file_size = os.path.getsize(file_path)
total_segments = max(1, (file_size + seg_max - 1) // seg_max) if file_size > 0 else 0
sha = hashlib.sha256()
segments: list[tuple[str, int]] = []
total_size = 0
with open(file_path, 'rb') as f:
while True:
buf = f.read(seg_max)
if not buf:
break
sha.update(buf)
total_size += len(buf)
seg_root = cls.compute_root(buf, chunk_size=chunk_size)
segments.append((seg_root, len(buf)))
if on_segment is not None:
on_segment(len(segments), max(len(segments), total_segments), total_size, file_size)
if total_size == 0:
empty_root = hashlib.sha256(b'\x00').hexdigest()
return sha.hexdigest(), empty_root, 0
tree_root = cls.combine_segment_roots(segments, chunk_size=chunk_size, subtree_pow=subtree_pow)
return sha.hexdigest(), tree_root, total_size
[docs] @classmethod
def verify(cls, data: bytes, expected_root: str, chunk_size: int = CHUNK_SIZE) -> bool:
"""Verify that data produces expected Merkle tree root."""
actual_root, _ = cls.compute_tree(data, chunk_size=chunk_size)
return actual_root.lower() == expected_root.lower().strip()
[docs]class RhioBlobSync(SqliteStore):
"""Metadata and CAS storage for Rhio cluster blobs.
A `SqliteStore` rather than a hand-rolled `sqlite3.connect`: the base owns
the connection, the lock, the schema version (per store, not per database -
which matters the moment two stores share one server), transaction control
and the dialect that rewrites `?` and `INSERT OR IGNORE` for PostgreSQL.
This store opened its own connection per call and declared its schema as a
SQL string, so it was SQLite-only and invisible to the locator - see
docs/storage-backends.md.
"""
SCHEMA = (
Table('rhio_blobs', (
Column('hash', 'TEXT', primary_key=True),
Column('size', 'INTEGER', not_null=True),
Column('tree_root', 'TEXT', not_null=True),
Column('synced_s3', 'INTEGER', not_null=True, default='0'),
Column('created_at', 'TEXT', not_null=True),
), indices=(
Index('idx_rhio_blobs_tree', ('tree_root',)),
)),
Table('rhio_fiber_claims', (
Column('fiber_root', 'TEXT', not_null=True),
Column('author_key', 'TEXT', not_null=True),
Column('payload_json', 'TEXT', not_null=True),
Column('created_at', 'INTEGER', not_null=True),
), primary_key=('fiber_root', 'author_key'), indices=(
Index('idx_rhio_fiber_claims_root', ('fiber_root',)),
Index('idx_rhio_fiber_claims_author', ('author_key',)),
)),
)
_EXTERNAL_FILE_THRESHOLD: int = 32 * 1024 * 1024 # 32 MiB
[docs] def __init__(
self,
db_path: Optional[str] = ":memory:",
cas_dir: Optional[str] = None,
overflow_cas_dirs: Optional[Sequence[str]] = None,
):
db_path = (db_path or ":memory:").strip() or ":memory:"
# A file-backed store with no cas_dir kept the bytes in a process dict
# while persisting the row that says it holds them. After a restart
# has_blob() was true, count_blobs() counted it, synced_s3 said the
# cluster already had a copy - and get_blob() returned nothing. Two of
# the three callers passed a directory; the video compartment did not,
# and nothing could tell, because every suite used `:memory:` where one
# connection reads back whatever was written.
#
# So derive it rather than trust each caller to remember: a persistent
# database gets a persistent CAS beside it. The dict stays for
# `:memory:` alone, where losing the bytes with the rows is honest.
if db_path != ":memory:":
os.makedirs(os.path.dirname(os.path.abspath(db_path)), exist_ok=True)
if cas_dir is None:
cas_dir = os.path.splitext(os.path.abspath(db_path))[0] + '_blobs'
self._cas_dir = cas_dir
self._mem_blobs: dict[str, bytes] = {}
self._external_files: dict[str, str] = {}
self._claims: dict[str, dict[str, dict[str, Any]]] = {}
self._by_target: dict[str, set[tuple[str, str]]] = {}
self._root_meta: dict[str, RhioBlobRecord] = {}
if cas_dir is not None:
os.makedirs(cas_dir, exist_ok=True)
raw_env_ov = (
os.environ.get('FLEETMESH_OVERFLOW_CAS_DIRS')
or os.environ.get('RHIO_OVERFLOW_CAS_DIRS')
or ''
).strip()
init_ov: list[str] = []
if overflow_cas_dirs:
init_ov.extend(str(d).strip() for d in overflow_cas_dirs if str(d).strip())
elif raw_env_ov:
for part in raw_env_ov.replace(',', ':').split(':'):
p = part.strip()
if p:
init_ov.append(p)
self._overflow_cas_dirs: list[str] = []
# Local files evict_local_file could not delete (permissions, a
# concurrent remover); the metadata row is kept either way.
self.evict_errors: int = 0
self._primary_min_free_bytes: int = int(
os.environ.get('FLEETMESH_PRIMARY_MIN_FREE_BYTES', str(5 * 1024 * 1024 * 1024))
)
super().__init__(db_path)
if init_ov:
self.set_overflow_cas_dirs(init_ov)
def _post_schema(self) -> None:
for h_raw, sz_raw, root_raw, s3_raw, cat_raw in self._all(
"SELECT hash, size, tree_root, synced_s3, created_at FROM rhio_blobs"
):
r = str(root_raw or '').strip().lower()
if r:
self._root_meta[r] = {
"hash": str(h_raw),
"size": int(sz_raw),
"tree_root": r,
"synced_s3": bool(s3_raw),
"created_at": str(cat_raw),
}
for root_raw, akey_raw, pjson in self._all(
"SELECT fiber_root, author_key, payload_json FROM rhio_fiber_claims"
):
try:
claim = json.loads(str(pjson))
except (TypeError, ValueError):
continue
root = str(root_raw or '').strip().lower()
akey = str(akey_raw or '').strip().lower()
if isinstance(claim, dict) and root and akey:
self._claims.setdefault(root, {})[akey] = claim
self._index_target(root, akey, claim)
[docs] def set_overflow_cas_dirs(self, dirs: Sequence[str]) -> list[str]:
"""Configure secondary/overflow CAS directories (e.g. attached block storage volumes)."""
cleaned: list[str] = []
primary_abs = os.path.abspath(self._cas_dir) if self._cas_dir else ''
with self._lock:
for raw_d in dirs:
d = str(raw_d or '').strip()
if not d:
continue
abs_d = os.path.abspath(d)
if abs_d == primary_abs or abs_d in cleaned:
continue
try:
os.makedirs(abs_d, exist_ok=True)
cleaned.append(abs_d)
except OSError:
continue
self._overflow_cas_dirs = cleaned
return list(cleaned)
[docs] def get_overflow_cas_dirs(self) -> list[str]:
with self._lock:
return list(self._overflow_cas_dirs)
def _select_target_cas_dir(self, required_bytes: int = 0) -> Optional[str]:
"""Return `self._cas_dir` when it has sufficient free space, or fail over to `_overflow_cas_dirs`."""
if not self._cas_dir:
return None
need = max(int(required_bytes) + 64 * 1024 * 1024, self._primary_min_free_bytes)
if not self._overflow_cas_dirs:
return self._cas_dir
try:
pri_free = shutil.disk_usage(self._cas_dir).free
if pri_free >= need:
return self._cas_dir
except OSError:
pri_free = 0
best_dir = self._cas_dir
best_free = pri_free
for ov_dir in list(self._overflow_cas_dirs):
try:
os.makedirs(ov_dir, exist_ok=True)
ov_free = shutil.disk_usage(ov_dir).free
if ov_free >= int(required_bytes) + 64 * 1024 * 1024 and ov_free > best_free:
return ov_dir
if ov_free > best_free:
best_free = ov_free
best_dir = ov_dir
except OSError:
continue
return best_dir
[docs] def store_blob(
self,
data: bytes,
*,
sha256_hex: Optional[str] = None,
tree_root: Optional[str] = None,
chunk_count: Optional[int] = None,
) -> tuple[str, str, int]:
"""Store blob, computing sha256 hash and Bao tree root if not provided.
Returns (sha256_hash, tree_root, size).
"""
blob_hash = sha256_hex.strip().lower() if sha256_hex else hashlib.sha256(data).hexdigest()
final_root = tree_root.strip().lower() if tree_root else RhioBaoTree.compute_root(data)
size = len(data)
_ = chunk_count
with self._lock:
# Save raw bytes
if self._cas_dir:
target_dir = self._select_target_cas_dir(required_bytes=size) or self._cas_dir
path = os.path.join(target_dir, blob_hash)
if not os.path.exists(path):
try:
with open(path, "wb") as f:
f.write(data)
except OSError:
for ov_dir in self._overflow_cas_dirs:
if ov_dir == target_dir:
continue
try:
ov_path = os.path.join(ov_dir, blob_hash)
with open(ov_path, "wb") as f:
f.write(data)
break
except OSError:
continue
else:
raise
else:
self._mem_blobs[blob_hash] = data
# INSERT OR IGNORE preserves synced_s3 and the original created_at.
# The dialect rewrites it for a backend that spells it differently.
created_iso = _now_iso()
with self.transaction() as cur:
cur.execute(
"""
INSERT OR IGNORE INTO rhio_blobs (hash, size, tree_root, synced_s3, created_at)
VALUES (?, ?, ?, 0, ?)
""",
(blob_hash, size, final_root, created_iso),
)
prev_meta = self._root_meta.get(final_root)
self._root_meta[final_root] = {
"hash": blob_hash,
"size": size,
"tree_root": final_root,
"synced_s3": bool(prev_meta.get("synced_s3")) if prev_meta else False,
"created_at": str(prev_meta.get("created_at") or created_iso) if prev_meta else created_iso,
}
return blob_hash, final_root, size
[docs] def store_blob_from_file(
self,
file_path: str,
*,
sha256_hex: Optional[str] = None,
tree_root: Optional[str] = None,
size: Optional[int] = None,
move: bool = False,
chunk_count: Optional[int] = None,
on_segment: Optional[Callable[[int, int, int, int], None]] = None,
) -> tuple[str, str, int]:
"""Store a file-backed blob into CAS without loading the full file into memory."""
_ = chunk_count
if not sha256_hex or not tree_root or size is None:
comp_sha, comp_root, comp_size = RhioBaoTree.compute_file_root(
file_path, on_segment=on_segment
)
blob_hash = (sha256_hex or comp_sha).lower().strip()
final_root = (tree_root or comp_root).lower().strip()
final_size = int(size if size is not None else comp_size)
else:
blob_hash = sha256_hex.lower().strip()
final_root = tree_root.lower().strip()
final_size = int(size)
if self._cas_dir:
with self._lock:
overflow_dirs = list(self._overflow_cas_dirs)
target_cas_dir = self._select_target_cas_dir(required_bytes=final_size) or self._cas_dir
dest_path = os.path.join(target_cas_dir, blob_hash)
src_abs = os.path.abspath(file_path)
dst_abs = os.path.abspath(dest_path)
if src_abs != dst_abs:
need_stage = True
existing_loc = self.get_blob_path(blob_hash)
if existing_loc and os.path.isfile(existing_loc):
try:
if os.path.getsize(existing_loc) == final_size:
need_stage = False
except OSError:
need_stage = True
if need_stage:
placed = False
cas_parent = os.path.dirname(os.path.abspath(self._cas_dir))
alt_candidates = [
os.path.join(cas_parent, 'uploads', os.path.basename(file_path)),
os.path.join(cas_parent, 'uploads', 'staging', os.path.basename(file_path)),
]
if move:
for cand_src in (file_path, *alt_candidates):
if not os.path.isfile(cand_src):
continue
try:
os.rename(cand_src, dest_path)
placed = True
if cand_src != file_path and os.path.exists(file_path):
try:
os.unlink(file_path)
except OSError:
pass
break
except OSError:
placed = False
else:
for cand_src in (file_path, *alt_candidates):
if not os.path.isfile(cand_src):
continue
for link_dir in (target_cas_dir, self._cas_dir, *overflow_dirs):
if not link_dir:
continue
cand_dest = os.path.join(link_dir, blob_hash)
try:
if os.path.getsize(cand_src) != final_size:
continue
if os.path.exists(cand_dest):
os.unlink(cand_dest)
os.link(cand_src, cand_dest)
dest_path = cand_dest
placed = True
break
except OSError:
placed = False
if placed:
break
if not placed:
candidate_dirs = [target_cas_dir] + [
d for d in (self._cas_dir, *overflow_dirs) if d and d != target_cas_dir
]
last_err: Optional[OSError] = None
for write_dir in candidate_dirs:
cand_dest = os.path.join(write_dir, blob_hash)
tmp_path = f"{cand_dest}.tmp.{os.getpid()}.{threading.get_ident()}"
try:
shutil.copyfile(file_path, tmp_path)
os.replace(tmp_path, cand_dest)
dest_path = cand_dest
placed = True
break
except OSError as exc:
last_err = exc
finally:
if os.path.exists(tmp_path):
try:
os.unlink(tmp_path)
except OSError:
pass
if not placed:
if not move and os.path.isfile(file_path):
with self._lock:
self._external_files[blob_hash] = os.path.abspath(file_path)
placed = True
elif last_err is not None:
raise last_err
if move and os.path.exists(file_path):
try:
os.unlink(file_path)
except OSError:
pass
elif move and os.path.exists(file_path):
try:
os.unlink(file_path)
except OSError:
pass
with self._lock:
if not self._cas_dir:
if final_size > self._EXTERNAL_FILE_THRESHOLD:
self._external_files[blob_hash] = os.path.abspath(file_path)
else:
with open(file_path, "rb") as f:
self._mem_blobs[blob_hash] = f.read()
if move and os.path.exists(file_path):
try:
os.unlink(file_path)
except OSError:
pass
created_iso = _now_iso()
with self.transaction() as cur:
cur.execute(
"""
INSERT OR IGNORE INTO rhio_blobs (hash, size, tree_root, synced_s3, created_at)
VALUES (?, ?, ?, 0, ?)
""",
(blob_hash, final_size, final_root, created_iso),
)
prev_meta = self._root_meta.get(final_root)
self._root_meta[final_root] = {
"hash": blob_hash,
"size": final_size,
"tree_root": final_root,
"synced_s3": bool(prev_meta.get("synced_s3")) if prev_meta else False,
"created_at": str(prev_meta.get("created_at") or created_iso) if prev_meta else created_iso,
}
return blob_hash, final_root, final_size
[docs] def get_blob_path(self, blob_hash: str) -> Optional[str]:
"""Return the filesystem path of a stored CAS blob if backed by `cas_dir`, `_overflow_cas_dirs`, or an external file."""
clean_hash = blob_hash.lower().strip()
if not clean_hash:
return None
with self._lock:
if self._cas_dir:
path = os.path.join(self._cas_dir, clean_hash)
if os.path.isfile(path):
return path
for ov_dir in self._overflow_cas_dirs:
ov_path = os.path.join(ov_dir, clean_hash)
if os.path.isfile(ov_path):
return ov_path
ext_path = self._external_files.get(clean_hash)
if ext_path and os.path.isfile(ext_path):
return ext_path
return None
[docs] def has_local_blob(self, blob_hash: str) -> bool:
"""Return True if the raw bytes for `blob_hash` are currently present on local/overflow disk or in memory."""
clean_hash = blob_hash.lower().strip()
if not clean_hash:
return False
if self.get_blob_path(clean_hash) is not None:
return True
with self._lock:
return clean_hash in self._mem_blobs
[docs] def migrate_blob_to_overflow(
self,
blob_hash: str,
target_dir: Optional[str] = None,
) -> Optional[str]:
"""Move `blob_hash` from primary `_cas_dir` to an overflow CAS volume and return the new path."""
clean_hash = blob_hash.lower().strip()
if not clean_hash or not self._cas_dir:
return None
pri_path = os.path.join(self._cas_dir, clean_hash)
if not os.path.isfile(pri_path):
return self.get_blob_path(clean_hash)
try:
fsize = os.path.getsize(pri_path)
except OSError:
return None
candidates = [target_dir] if target_dir else list(self._overflow_cas_dirs)
for ov_dir in candidates:
if not ov_dir:
continue
abs_ov = os.path.abspath(ov_dir)
if abs_ov == os.path.abspath(self._cas_dir):
continue
try:
os.makedirs(abs_ov, exist_ok=True)
if shutil.disk_usage(abs_ov).free < fsize + 32 * 1024 * 1024:
continue
dest_path = os.path.join(abs_ov, clean_hash)
if not os.path.isfile(dest_path) or os.path.getsize(dest_path) != fsize:
tmp_path = f"{dest_path}.mig.{os.getpid()}.{threading.get_ident()}"
try:
shutil.copyfile(pri_path, tmp_path)
os.replace(tmp_path, dest_path)
finally:
if os.path.exists(tmp_path):
try:
os.unlink(tmp_path)
except OSError:
pass
if os.path.isfile(dest_path) and os.path.getsize(dest_path) == fsize:
with self._lock:
if abs_ov not in self._overflow_cas_dirs:
self._overflow_cas_dirs.append(abs_ov)
try:
os.unlink(pri_path)
except OSError:
pass
return dest_path
except OSError:
continue
return None
def _remove_file(self, path: str) -> int:
"""Delete `path` and return the bytes freed; a failure is counted in evict_errors."""
try:
size = os.path.getsize(path)
os.unlink(path)
except OSError:
self.evict_errors += 1
return 0
return size
[docs] def evict_local_file(self, blob_hash: str, *, include_overflow: bool = False) -> int:
"""Evict the local file for `blob_hash` to free disk space while preserving its `rhio_blobs` metadata row."""
clean_hash = blob_hash.lower().strip()
if not clean_hash:
return 0
freed_bytes = 0
with self._lock:
dirs_to_check = [self._cas_dir] if self._cas_dir else []
if include_overflow:
dirs_to_check.extend(self._overflow_cas_dirs)
for d in dirs_to_check:
if not d:
continue
path = os.path.join(d, clean_hash)
if os.path.isfile(path):
freed_bytes += self._remove_file(path)
if clean_hash in self._mem_blobs:
freed_bytes += len(self._mem_blobs.pop(clean_hash))
ext_path = self._external_files.pop(clean_hash, None)
if ext_path and os.path.isfile(ext_path):
freed_bytes += self._remove_file(ext_path)
return freed_bytes
[docs] def get_blob(self, blob_hash: str) -> Optional[bytes]:
"""Retrieve blob bytes by sha256 hash."""
clean_hash = blob_hash.lower().strip()
path = self.get_blob_path(clean_hash)
if path and os.path.isfile(path):
with open(path, "rb") as f:
return f.read()
with self._lock:
if clean_hash in self._mem_blobs:
return self._mem_blobs[clean_hash]
return None
[docs] def get_blob_slice(self, blob_hash: str, start: int, length: int) -> Optional[bytes]:
"""Read a byte range `[start : start + length]` directly from CAS disk without loading the full blob into RAM."""
clean_hash = blob_hash.lower().strip()
if start < 0 or length < 0:
return None
path = self.get_blob_path(clean_hash)
if path and os.path.isfile(path):
with open(path, "rb") as f:
f.seek(start)
return f.read(length)
with self._lock:
raw = self._mem_blobs.get(clean_hash)
if raw is None:
return None
return raw[start : start + length]
[docs] def get_blob_path_by_root(self, tree_root: str) -> Optional[str]:
"""Return the CAS filesystem path for the blob matching `tree_root`."""
meta = self.get_blob_meta_by_root(tree_root)
return self.get_blob_path(meta["hash"]) if meta else None
[docs] def get_blob_by_root(self, tree_root: str) -> Optional[bytes]:
"""Retrieve blob bytes by `tree_root`."""
meta = self.get_blob_meta_by_root(tree_root)
return self.get_blob(meta["hash"]) if meta else None
[docs] def get_blob_slice_by_root(self, tree_root: str, start: int, length: int) -> Optional[bytes]:
"""Read a byte range `[start : start + length]` by `tree_root` without loading the full blob into RAM."""
meta = self.get_blob_meta_by_root(tree_root)
return self.get_blob_slice(meta["hash"], start, length) if meta else None
[docs] def delete_blob(self, blob_hash: str) -> bool:
"""Delete an ephemeral blob from CAS disk/memory and metadata store."""
clean_hash = blob_hash.lower().strip()
if not clean_hash:
return False
removed = False
with self._lock:
meta = self.get_blob_meta(clean_hash)
root_to_check = str(meta["tree_root"]).lower() if meta else ""
for d in ([self._cas_dir] if self._cas_dir else []) + list(self._overflow_cas_dirs):
if not d:
continue
path = os.path.join(d, clean_hash)
if os.path.exists(path):
try:
os.unlink(path)
except OSError:
pass
else:
removed = True
if clean_hash in self._mem_blobs:
del self._mem_blobs[clean_hash]
removed = True
if clean_hash in self._external_files:
del self._external_files[clean_hash]
removed = True
with self.transaction() as cur:
cur.execute("DELETE FROM rhio_blobs WHERE hash = ?", (clean_hash,))
stale_roots = [
r for r, rm in self._root_meta.items() if rm.get("hash") == clean_hash
]
if root_to_check and root_to_check not in stale_roots:
stale_roots.append(root_to_check)
for r_del in stale_roots:
self._root_meta.pop(r_del, None)
if self._one("SELECT 1 FROM rhio_blobs WHERE tree_root = ?", (r_del,)) is None:
old_claims = self._claims.pop(r_del, {})
for akey, clm in old_claims.items():
self._unindex_target(r_del, akey, clm)
cur.execute("DELETE FROM rhio_fiber_claims WHERE fiber_root = ?", (r_del,))
return removed
[docs] def has_blob(self, blob_hash: str) -> bool:
"""Check if blob exists in storage."""
clean_hash = blob_hash.lower().strip()
with self._lock:
if clean_hash in self._external_files and os.path.isfile(self._external_files[clean_hash]):
return True
return self._one("SELECT 1 FROM rhio_blobs WHERE hash = ?", (clean_hash,)) is not None
[docs] def mark_s3_synced(self, blob_hash: str) -> None:
"""Mark blob as synchronized with S3 / MinIO cluster storage."""
clean_hash = blob_hash.lower().strip()
with self._lock:
with self.transaction() as cur:
cur.execute("UPDATE rhio_blobs SET synced_s3 = 1 WHERE hash = ?", (clean_hash,))
for rmeta in self._root_meta.values():
if rmeta.get("hash") == clean_hash:
rmeta["synced_s3"] = True
[docs] def count_blobs(self) -> int:
with self._lock:
row = self._one("SELECT COUNT(*) FROM rhio_blobs")
return int(row[0]) if row else 0
[docs] def list_blobs(self, limit: int = 100) -> list[RhioBlobRecord]:
with self._lock:
rows = self._all(
"SELECT hash, size, tree_root, synced_s3, created_at FROM rhio_blobs "
"ORDER BY created_at DESC LIMIT ?",
(limit,),
)
return [
{
"hash": str(r[0]),
"size": int(r[1]),
"tree_root": str(r[2]),
"synced_s3": bool(r[3]),
"created_at": str(r[4]),
}
for r in rows
]
# ---- Unified Content-Addressed Fiber Claims -----------------------------
def _index_target(self, root: str, key: str, claim: Mapping[str, Any]) -> None:
target = str((claim.get('target') or {}).get('strand_id') or '')
if target:
self._by_target.setdefault(target, set()).add((root, key))
def _unindex_target(self, root: str, key: str, claim: Mapping[str, Any]) -> None:
target = str((claim.get('target') or {}).get('strand_id') or '')
refs = self._by_target.get(target)
if refs is not None:
refs.discard((root, key))
if not refs:
del self._by_target[target]
def _viewer_claim(self, root: str, principal: Optional[Mapping[str, Any]]) -> Optional[dict[str, Any]]:
"""The caller's own claim, else the earliest public one; `None` means not readable."""
claims = self._claims.get(root) or {}
own = claims.get(_principal_key(principal)) if principal else None
if own is not None:
return own
public = [c for c in claims.values() if c.get('visibility') == 'public']
return min(public, key=lambda c: _safe_int(c.get('created_at'))) if public else None
def _manifest(
self,
root: str,
claim: Mapping[str, Any],
meta: Optional[RhioBlobRecord] = None,
) -> dict[str, Any]:
if meta is None:
meta = self.get_blob_meta_by_root(root)
size = int(meta["size"]) if meta else 0
sha = str(meta["hash"]) if meta else ""
out: dict[str, Any] = {
'fiber_root': root,
'sha256_hex': sha,
'byte_length': size,
'leaf_count_1024b': RhioBaoTree.leaf_count(size),
'kind': claim.get('kind'),
'author': dict(claim.get('author') or {}),
'visibility': claim.get('visibility'),
'mime_type': claim.get('mime_type') or '',
'created_at': claim.get('created_at'),
}
if claim.get('target'):
out['target'] = dict(claim['target'])
return out
[docs] def has_fiber(self, root: str) -> bool:
clean_root = str(root or '').strip().lower()
if not clean_root:
return False
with self._lock:
return bool(self._claims.get(clean_root)) and self.get_blob_meta_by_root(clean_root) is not None
[docs] def readable(self, root: str, principal: Optional[Mapping[str, Any]] = None) -> bool:
return self.manifest(root, principal) is not None
[docs] def manifest(self, root: str, principal: Optional[Mapping[str, Any]] = None) -> Optional[dict[str, Any]]:
clean_root = str(root or '').strip().lower()
if not clean_root:
return None
with self._lock:
meta = self.get_blob_meta_by_root(clean_root)
if meta is None:
return None
claim = self._viewer_claim(clean_root, principal)
return self._manifest(clean_root, claim, meta) if claim else None
[docs] def get(self, root: str, principal: Optional[Mapping[str, Any]] = None) -> Optional[tuple[bytes, dict[str, Any]]]:
clean_root = str(root or '').strip().lower()
if not clean_root:
return None
with self._lock:
m = self.manifest(clean_root, principal)
if m is None:
return None
raw = self.get_blob(str(m.get('sha256_hex') or ''))
return (raw, m) if raw is not None else None
[docs] def list_own(self, principal: Optional[Mapping[str, Any]]) -> list[dict[str, Any]]:
key = _principal_key(principal)
if not key:
return []
with self._lock:
out: list[dict[str, Any]] = []
for root, claims in self._claims.items():
if key in claims:
meta = self.get_blob_meta_by_root(root)
if meta is not None:
out.append(self._manifest(root, claims[key], meta))
return sorted(out, key=lambda m: _safe_int(m.get('created_at')), reverse=True)
[docs] def owner_bytes(self, principal: Optional[Mapping[str, Any]]) -> int:
key = _principal_key(principal)
if not key:
return 0
with self._lock:
total = 0
for root, claims in self._claims.items():
if key in claims:
meta = self.get_blob_meta_by_root(root)
if meta is not None:
total += int(meta["size"])
return total
[docs] def interaction_counts(self, target_strand_id: str) -> dict[str, int]:
"""Public interaction claims on a strand, by kind (each actor's like counts once)."""
counts: dict[str, int] = {}
with self._lock:
for root, key in self._by_target.get(target_strand_id, ()):
claim = (self._claims.get(root) or {}).get(key) or {}
kind = str(claim.get('kind') or '')
if claim.get('visibility') == 'public' and kind in INTERACTION_KINDS:
counts[kind] = counts.get(kind, 0) + 1
return counts
[docs] def claim_blob(
self,
tree_root: str,
principal: Optional[Mapping[str, Any]],
*,
sha256_hex: str = '',
size: Optional[int] = None,
kind: str = 'binary',
mime_type: str = '',
visibility: str = 'public',
target_strand_id: str = '',
overwrite_visibility: bool = True,
created_at: Optional[int] = None,
) -> Optional[dict[str, Any]]:
"""Register or update an author's fiber claim on an existing (or metadata-registered) blob."""
clean_root = str(tree_root or '').strip().lower()
if len(clean_root) != 64:
return None
who = principal if (principal and _principal_key(principal)) else {'identity_id': 'node', 'did': '', 'handle': ''}
key = _principal_key(who) or 'node'
vis = 'vault_only' if visibility == 'vault_only' else 'public'
eff_kind = kind if kind in FIBER_KINDS else 'binary'
now = int(created_at if created_at is not None else time.time())
with self._lock:
meta = self.get_blob_meta_by_root(clean_root)
if meta is None and sha256_hex and size is not None:
clean_sha = str(sha256_hex).strip().lower()
if len(clean_sha) == 64:
with self.transaction() as cur:
cur.execute(
"""
INSERT OR IGNORE INTO rhio_blobs (hash, size, tree_root, synced_s3, created_at)
VALUES (?, ?, ?, 0, ?)
""",
(clean_sha, int(size), clean_root, _now_iso()),
)
meta = self.get_blob_meta_by_root(clean_root)
if meta is None:
return None
claims = self._claims.setdefault(clean_root, {})
existing = claims.get(key)
claim = dict(existing or {'author': _author_ref(who), 'kind': eff_kind, 'created_at': now})
if existing is None or overwrite_visibility:
claim['visibility'] = vis
if mime_type or not claim.get('mime_type'):
claim['mime_type'] = str(mime_type or claim.get('mime_type') or '')[:120]
if kind in FIBER_KINDS:
claim['kind'] = kind
if target_strand_id:
if existing is not None:
self._unindex_target(clean_root, key, existing)
claim['target'] = {'strand_id': str(target_strand_id)[:200]}
claims[key] = claim
self._index_target(clean_root, key, claim)
pjson = json.dumps(claim, separators=(',', ':'))
with self.transaction() as cur:
cur.execute(
"""
INSERT OR REPLACE INTO rhio_fiber_claims (fiber_root, author_key, payload_json, created_at)
VALUES (?, ?, ?, ?)
""",
(clean_root, key, pjson, _safe_int(claim.get('created_at'), now)),
)
return self._manifest(clean_root, claim, meta)
[docs] def put(
self,
raw: bytes,
principal: Optional[Mapping[str, Any]],
*,
kind: str,
mime_type: str = '',
visibility: str = 'vault_only',
target_strand_id: str = '',
enforce_quota: bool = True,
) -> dict[str, Any]:
"""Store `raw` (once) in `RhioBlobSync` and record the caller's claim on it; returns the caller's manifest."""
who = _require_principal(principal)
if kind not in FIBER_KINDS:
raise FiberAccessError(400, 'invalid_fiber_kind')
if kind in INTERACTION_KINDS and not target_strand_id:
raise FiberAccessError(400, 'interaction_target_required')
if len(raw) > MAX_FIBER_BYTES:
raise FiberAccessError(413, 'binary_too_large')
vis = visibility if visibility in VISIBILITIES else 'vault_only'
root = RhioBaoTree.compute_root(raw)
key = _principal_key(who)
now = int(time.time())
with self._lock:
existing = (self._claims.get(root) or {}).get(key)
if existing is None and enforce_quota and self.owner_bytes(who) + len(raw) > MAX_OWNER_BYTES:
raise FiberAccessError(413, 'owner_quota_exceeded')
sha, _, size = self.store_blob(raw, tree_root=root)
manifest = self.claim_blob(
root,
who,
sha256_hex=sha,
size=size,
kind=kind,
mime_type=mime_type,
visibility=vis,
target_strand_id=target_strand_id,
overwrite_visibility=True,
created_at=now,
)
assert manifest is not None
return manifest
[docs] def set_visibility(self, root: str, principal: Optional[Mapping[str, Any]], visibility: str) -> dict[str, Any]:
who = _require_principal(principal)
key = _principal_key(who)
clean_root = str(root or '').strip().lower()
if visibility not in VISIBILITIES:
raise FiberAccessError(400, 'invalid_visibility')
with self._lock:
claim = (self._claims.get(clean_root) or {}).get(key)
meta = self.get_blob_meta_by_root(clean_root) if claim is not None else None
if claim is None or meta is None:
raise FiberAccessError(404, 'fiber_claim_not_found')
if claim.get('visibility') != visibility:
claim['visibility'] = visibility
pjson = json.dumps(claim, separators=(',', ':'))
with self.transaction() as cur:
cur.execute(
"UPDATE rhio_fiber_claims SET payload_json = ? WHERE fiber_root = ? AND author_key = ?",
(pjson, clean_root, key),
)
return self._manifest(clean_root, claim, meta)
[docs] def release(self, root: str, principal: Optional[Mapping[str, Any]]) -> bool:
"""Drop the caller's claim; the underlying blob is deleted when no claims remain."""
who = _require_principal(principal)
key = _principal_key(who)
clean_root = str(root or '').strip().lower()
with self._lock:
claims = self._claims.get(clean_root) or {}
if key not in claims:
return False
self._unindex_target(clean_root, key, claims.pop(key))
with self.transaction() as cur:
cur.execute(
"DELETE FROM rhio_fiber_claims WHERE fiber_root = ? AND author_key = ?",
(clean_root, key),
)
if not claims:
self._claims.pop(clean_root, None)
meta = self.get_blob_meta_by_root(clean_root)
if meta is not None:
self.delete_blob(meta["hash"])
return True
[docs] def close(self) -> None:
super().close()
[docs]def binary_http_response(
raw: bytes,
*,
bao_root: str,
sha256_hex: str,
mime_type: str,
public: bool,
filename: str,
as_json: bool,
json_extra: Optional[Mapping[str, Any]] = None,
) -> tuple[int, Any, str, dict[str, str]]:
"""Serve stored bytes so they can never render as an active page on the app's origin."""
cache = 'public, max-age=3600, immutable' if public else 'private, no-store'
hdrs: dict[str, str] = {
'X-Bao-Tree-Root': bao_root,
'Cache-Control': cache,
'X-Content-Type-Options': 'nosniff',
'Accept-Ranges': 'bytes',
}
mime = str(mime_type or '').strip().lower()
if as_json:
return 200, {
'ok': True,
**dict(json_extra or {}),
'bao_root': bao_root,
'sha256_hex': sha256_hex,
'mime_type': mime or 'application/octet-stream',
'size_bytes': len(raw),
'data_b64': base64.b64encode(raw).decode('ascii'),
}, JSON_CT, hdrs
fname = re.sub(r'[^A-Za-z0-9._-]', '_', filename or 'fiber.bin')[:120]
hdrs['Content-Disposition'] = f'attachment; filename="{fname}"'
hdrs['Content-Security-Policy'] = "default-src 'none'; sandbox"
return 200, raw, (mime if mime in _SAFE_BINARY_MIME else 'application/octet-stream'), hdrs
[docs]def is_fiber_route(route: str) -> bool:
return route == '/api/fibers' or route.startswith('/api/fibers/')
def _not_found(root: str) -> tuple[int, Any, str, dict[str, str]]:
return 404, {'ok': False, 'error': 'fiber_not_found', 'fiber_root': root}, JSON_CT, {}
[docs]def handle_fiber_http(
store: Any,
method: str,
route: str,
query: Any = '',
body: Optional[Mapping[str, Any]] = None,
principal: Optional[Mapping[str, Any]] = None,
*,
range_header: str = '',
) -> Optional[tuple[int, Any, str, dict[str, str]]]:
"""Route `/api/fibers*`. `principal` must come from the server's own session resolution."""
store = getattr(store, 'fiber_store', getattr(store, 'rhio_store', store))
qp = query if isinstance(query, Mapping) else urllib.parse.parse_qs(str(query or ''))
try:
if method in ('GET', 'HEAD'):
if route == '/api/fibers':
if not principal or not _principal_key(principal):
raise FiberAccessError(401, 'authentication_required')
return 200, {'ok': True, 'fibers': store.list_own(principal)}, JSON_CT, {'Cache-Control': 'private, no-store'}
if route == '/api/fibers/interactions':
target = (qp.get('target', [''])[0] or '').strip()
if not target:
raise FiberAccessError(400, 'target_required')
return 200, {'ok': True, 'target': target, 'counts': store.interaction_counts(target)}, JSON_CT, {'Cache-Control': 'public, max-age=15'}
parts = [p for p in route[len('/api/fibers/'):].split('/') if p]
if not parts or len(parts) > 2 or not _ROOT_RE.match(parts[0]):
return None
root = parts[0]
if len(parts) == 2:
if parts[1] != 'manifest':
return None
manifest = store.manifest(root, principal)
return (200, {'ok': True, 'fiber': manifest}, JSON_CT, {'Cache-Control': 'private, no-store'}) if manifest else _not_found(root)
as_json = (qp.get('format', [''])[0] or '').strip().lower() == 'json'
rng = str(range_header or '').strip()
if method == 'HEAD' and not rng and not as_json:
manifest = store.manifest(root, principal)
if manifest is None:
return _not_found(root)
_st, _body, ct, hdrs = binary_http_response(
b'',
bao_root=root,
sha256_hex=manifest['sha256_hex'],
mime_type=manifest.get('mime_type') or '',
public=manifest.get('visibility') == 'public',
filename=f"{root[:16]}.bin",
as_json=False,
)
hdrs['Content-Length'] = str(int(manifest.get('byte_length') or 0))
return 200, b'', ct, hdrs
if rng and not as_json:
manifest = store.manifest(root, principal)
if manifest is None:
return _not_found(root)
total = int(manifest.get('byte_length') or 0)
m = _RANGE_HEADER_RE.match(rng)
if not m:
return 416, {'ok': False, 'error': 'range_not_satisfiable'}, JSON_CT, {'Content-Range': f'bytes */{total}', 'Accept-Ranges': 'bytes'}
start_s, end_s = m.group(1), m.group(2)
if start_s == '' and end_s == '':
return 416, {'ok': False, 'error': 'range_not_satisfiable'}, JSON_CT, {'Content-Range': f'bytes */{total}', 'Accept-Ranges': 'bytes'}
if start_s == '' and end_s != '':
suffix_len = int(end_s)
start = max(0, total - suffix_len)
end = total - 1
else:
start = int(start_s) if start_s else 0
end = int(end_s) if end_s else (total - 1)
if total <= 0 or start >= total or start > end:
return 416, {'ok': False, 'error': 'range_not_satisfiable'}, JSON_CT, {'Content-Range': f'bytes */{total}', 'Accept-Ranges': 'bytes'}
end = min(end, total - 1)
slice_len = end - start + 1
if method == 'HEAD':
_st, _body, ct, hdrs = binary_http_response(
b'',
bao_root=root,
sha256_hex=manifest['sha256_hex'],
mime_type=manifest.get('mime_type') or '',
public=manifest.get('visibility') == 'public',
filename=f"{root[:16]}.bin",
as_json=False,
)
hdrs['Content-Range'] = f'bytes {start}-{end}/{total}'
hdrs['Content-Length'] = str(slice_len)
hdrs['X-Bao-Range-Root'] = root
return 206, b'', ct, hdrs
if hasattr(store, 'get_blob_slice') and manifest.get('sha256_hex'):
chunk = store.get_blob_slice(str(manifest['sha256_hex']), start, slice_len)
elif hasattr(store, 'get_blob_slice_by_root'):
chunk = store.get_blob_slice_by_root(root, start, slice_len)
else:
found_full = store.get(root, principal)
chunk = found_full[0][start : end + 1] if found_full else None
if chunk is None:
return _not_found(root)
_st, body_b, ct, hdrs = binary_http_response(
chunk,
bao_root=root,
sha256_hex=manifest['sha256_hex'],
mime_type=manifest.get('mime_type') or '',
public=manifest.get('visibility') == 'public',
filename=f"{root[:16]}.bin",
as_json=False,
)
hdrs['Content-Range'] = f'bytes {start}-{end}/{total}'
if len(chunk) <= 1024 * 1024:
hdrs['X-Bao-Range-Root'] = RhioBaoTree.compute_root(chunk)
else:
hdrs['X-Bao-Range-Root'] = root
return 206, body_b, ct, hdrs
found = store.get(root, principal)
if found is None:
return _not_found(root)
raw, manifest = found
return binary_http_response(
raw,
bao_root=root,
sha256_hex=manifest['sha256_hex'],
mime_type=manifest.get('mime_type') or '',
public=manifest.get('visibility') == 'public',
filename=f"{root[:16]}.bin",
as_json=as_json,
json_extra={'fiber': manifest},
)
if method != 'POST' or not is_fiber_route(route):
return None
payload: Mapping[str, Any] = body or {}
if route in ('/api/fibers', '/api/fibers/put'):
raw = _decode_b64(payload.get('data_b64'), MAX_FIBER_BYTES)
if raw is None:
raise FiberAccessError(400, 'data_b64_required')
manifest = store.put(
raw,
principal,
kind=str(payload.get('kind') or ''),
mime_type=str(payload.get('mime_type') or ''),
visibility=str(payload.get('visibility') or 'vault_only'),
target_strand_id=str(payload.get('target_strand_id') or ''),
)
return 200, {'ok': True, 'fiber': manifest}, JSON_CT, {}
if route == '/api/fibers/visibility':
manifest = store.set_visibility(
str(payload.get('fiber_root') or ''),
principal,
str(payload.get('visibility') or ''),
)
return 200, {'ok': True, 'fiber': manifest}, JSON_CT, {}
if route == '/api/fibers/release':
root = str(payload.get('fiber_root') or '')
released = store.release(root, principal)
return (200 if released else 404), {'ok': released, 'fiber_root': root}, JSON_CT, {}
return None
except Exception as exc:
status = getattr(exc, 'status', None)
error = getattr(exc, 'error', None)
if isinstance(status, int) and isinstance(error, str):
return status, {'ok': False, 'error': error}, JSON_CT, {}
raise