Source code for fleetmesh_core.storage.linked_data

"""Linked data: schema.org projections of the documents a node publishes for others to find.

A release, a paid-fetch quote, a blob descriptor and a node descriptor are
each already a typed structure with a wire form that some other standard
fixes (npack kind 9900, the `paid-fetch-402` body, Blossom BUD-02, kind
11801). Those stay the source of truth: signed, byte-exact, unchanged here.

What this module adds is a second reading of each, in the vocabulary at
https://schema.org, as JSON-LD - plain dicts with an `@context`. A consumer
that has never read fleetmesh's documents but knows what an `Offer` with a
`priceSpecification` is, or a `SoftwareApplication` with a `softwareVersion`,
can read what a node says without us in between. That is the whole point;
nothing in the protocol depends on these.

Rules, so the two readings never drift:

  - Projections only for public descriptive documents. Operator state,
    configuration and protocol wire bodies are not projected; schema.org has
    no words for them and nobody outside the node reads them.
  - Every term emitted exists in the pinned vocabulary
    (schema/schemaorg-terms.json, `version`; the pin also names the one
    `@context` emitted, which the unplug scan reads from there - a node
    reaches nothing). `terms()` lists what a
    projection used; the suite and fleetmesh.org's checker hold it against
    the pin, so a typo or a deprecated term never ships.
  - Nothing is invented. A projection carries only facts the source carries;
    where schema.org has no exact slot the fact goes under `additionalProperty`
    as a `PropertyValue` whose `propertyID` is the fleetmesh field name, so
    the mapping is visible in the output itself.
  - Money is in millisatoshi and schema.org prices are in a currency. The
    `price` is in BTC as a decimal string (schema.org names the ticker form
    for cryptocurrencies) and the exact msat rides beside it in a
    `PropertyValue` with `propertyID` `price_msat`. A reader that honours the
    convention loses nothing; one that does not still gets the right order
    of magnitude in a unit it knows.
  - Times are Unix seconds in the sources and ISO 8601 UTC in schema.org;
    `iso()` is the one conversion.

Standard library only. Nothing here reads the network or the wall clock.
"""
from __future__ import annotations

import datetime as _dt
from typing import Any, Mapping, MutableMapping, Optional, Sequence, TypedDict

from ..accounting.paidfetch import ItemBody, QuoteBody
from ..identity import relaylist
from ..peering.release import ReleaseDict
from .blossom_auth import BlobDescriptor
from ..wire import schema_files
from ..wire.interfaces import EventLike, StandingTellBody

# A JSON-LD node: an open map because the vocabulary, not this module, says
# what keys it may have - the pin checks the keys, not a type.
LinkedData = MutableMapping[str, Any]

CONTEXT: str = 'https://schema.org'   # == vocabulary()['context']; test_linked_data holds them equal

MSAT_PER_BTC: int = 100_000_000_000


[docs]def iso(unix: int) -> str: """Unix seconds as ISO 8601 UTC, the form schema.org's Date/DateTime take.""" return _dt.datetime.fromtimestamp(int(unix), tz=_dt.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
[docs]def btc(msat: int) -> str: """msat as a decimal BTC string with no float on the way: 1 msat is '0.00000000001'.""" msat = int(msat) whole, frac = divmod(msat, MSAT_PER_BTC) return '%d.%011d' % (whole, frac)
def _did(pubkey: str) -> str: return 'did:nostr:%s' % pubkey def _prop(prop_id: str, value: Any, unit: Optional[str] = None) -> LinkedData: """A fleetmesh field with no schema.org slot, named by the field it came from.""" pv: LinkedData = {'@type': 'PropertyValue', 'propertyID': prop_id, 'value': value} if unit: pv['unitText'] = unit return pv # --- release (kind 9900) -> SoftwareApplication ---------------------------------
[docs]def software_application(release: ReleaseDict | Mapping[str, Any]) -> LinkedData: """A `release.ReleaseDict` as a schema:SoftwareApplication. name/version/os/arch have exact slots. The artifact digest is the `sha256` of a MediaObject in `associatedMedia` (a release names the artifact's hash, not a URL). The signer is the `author`, as a did:nostr. `protocols` and `artifact_event` have no slot and are kept by name. """ doc: LinkedData = { '@context': CONTEXT, '@type': 'SoftwareApplication', 'identifier': release['event_id'], 'name': release['name'], 'softwareVersion': release['version'], 'operatingSystem': release['os'], 'processorRequirements': release['arch'], 'dateCreated': iso(release['created_at']), 'author': {'@type': 'Person', 'identifier': _did(release['signer'])}, 'associatedMedia': {'@type': 'MediaObject', 'sha256': release['sha256']}, } extra = [_prop('protocols', list(release.get('protocols') or []))] if release.get('artifact_event'): extra.append(_prop('artifact_event', release['artifact_event'])) doc['additionalProperty'] = extra return doc
# --- paid-fetch quote (402 body) -> Offer ----------------------------------------
[docs]def offer(quote: QuoteBody | Mapping[str, Any], url: Optional[str] = None) -> LinkedData: """A `paidfetch.QuoteBody` as a schema:Offer for one blob. The item offered is the blob as a MediaObject (sha256, contentSize in bytes). The price is a CompoundPriceSpecification: one component per `items[]` entry, each naming who it is for (`to`, `payee`) and carrying its msat exactly. `offers[]` rails become `acceptedPaymentMethod` texts (`cashu`, ...); their requests are rail-specific and not projected. """ items: Sequence[ItemBody | Mapping[str, Any]] = quote.get('items') or [] total = int(quote.get('price_msat', sum(int(i['msat']) for i in items))) components = [] for i in items: components.append({ '@type': 'UnitPriceSpecification', 'price': btc(i['msat']), 'priceCurrency': 'BTC', 'name': i['to'], 'additionalProperty': [ _prop('price_msat', int(i['msat']), 'msat'), _prop('payee', _did(i['payee'])), ], }) item: LinkedData = {'@type': 'MediaObject', 'sha256': quote['sha256'], 'contentSize': str(int(quote['size']))} if url: item['contentUrl'] = url doc: LinkedData = { '@context': CONTEXT, '@type': 'Offer', 'itemOffered': item, 'price': btc(total), 'priceCurrency': 'BTC', 'priceSpecification': { '@type': 'CompoundPriceSpecification', 'price': btc(total), 'priceCurrency': 'BTC', 'priceComponent': components, 'additionalProperty': [_prop('price_msat', total, 'msat')], }, 'validThrough': iso(quote['expires']), } rails = sorted({str(o.get('rail')) for o in (quote.get('offers') or []) if o.get('rail')}) if rails: doc['acceptedPaymentMethod'] = rails if url: doc['url'] = url return doc
# --- Blossom blob descriptor (BUD-02) -> MediaObject -----------------------------
[docs]def media_object(blob: BlobDescriptor | Mapping[str, Any]) -> LinkedData: """A `blossom_auth.BlobDescriptor` as a schema:MediaObject. Every BUD-02 field has an exact slot: url -> contentUrl, sha256 -> sha256, size -> contentSize (bytes), type -> encodingFormat, uploaded -> uploadDate. An IPFS `cid`, when present, is a second `identifier`. """ doc: LinkedData = {'@context': CONTEXT, '@type': 'MediaObject', 'sha256': blob['sha256']} if blob.get('url'): doc['contentUrl'] = blob['url'] if 'size' in blob: doc['contentSize'] = str(int(blob['size'])) if blob.get('type'): doc['encodingFormat'] = blob['type'] if 'uploaded' in blob: doc['uploadDate'] = iso(blob['uploaded']) if blob.get('cid'): doc['identifier'] = [{'@type': 'PropertyValue', 'propertyID': 'cid', 'value': blob['cid']}] return doc
# --- node descriptor (kind 11801) -> Service ------------------------------------- # An endpoint's scheme names the transport rung that dials it, in the names # replication.negotiate() ranks. The relay rung is the store-and-forward # floor of the ladder: every node has it, it is tried last, and it is the one # a stranger can always reach a node on. It is a rung - how a node transmits - # not a class of node; `relay` among the capabilities is a different fact # (this node HOSTS a relay for others). RUNG_OF_SCHEME: Mapping[str, str] = {'fips': 'fips-native', 'iroh': 'iroh', 'libp2p': 'libp2p', 'wss': 'nostr', 'ws': 'nostr'} FLOOR_RUNG: str = 'nostr'
[docs]def rung_of(scheme: str) -> str: return RUNG_OF_SCHEME.get(scheme, scheme)
def _tags(desc: EventLike, name: str) -> list[list[str]]: return [t for t in desc.get('tags', []) if t and t[0] == name]
[docs]def service(desc: EventLike, inbox: Optional[EventLike] = None) -> LinkedData: """A signed kind 11801 - and the node's kind 10050, when the caller has it - as a schema:Service. The node is the Service, identified by its did:nostr; the operator is the `provider`. `availableChannel` is the transport ladder: one ServiceChannel per `endpoint` tag, in the order the router walks them (preference, low first), each naming its scheme, its `rung` and its `preference`; then, last, the floor: one ServiceChannel per inbox relay the node's kind 10050 names (`relaylist.dm_relays`), rung `nostr`, `floor: true` - the store-and-forward channel every node has and tries last. The floor is read from the 10050 and not from the descriptor because that is where the protocol keeps it; an `endpoint wss` on a descriptor says the node *hosts* a relay, a different fact, and is projected as such (rung `nostr`, no floor bit). `serviceType` is the node class (leaf/edge/anchor), which declares connectivity, never capability; `c` tags (protocol identifiers and capabilities such as hosting a relay) are kept by name, as are the policy digest and software. `expiration` -> `expires`. """ tags = desc.get('tags', []) first = {t[0]: t for t in reversed(tags) if t} eps = _tags(desc, 'endpoint') eps = sorted(eps, key=lambda t: (int(t[3]) if len(t) > 3 and str(t[3]).isdigit() else 50, t[1])) channels = [] for t in eps: rung = rung_of(t[1]) ch: LinkedData = {'@type': 'ServiceChannel', 'name': t[1], 'serviceUrl': t[2]} extra_ch = [_prop('rung', rung)] if len(t) > 3 and str(t[3]).isdigit(): extra_ch.append(_prop('preference', int(t[3]))) ch['additionalProperty'] = extra_ch channels.append(ch) if inbox is not None and inbox.get('pubkey') == desc.get('pubkey'): for url in relaylist.dm_relays(inbox): channels.append({'@type': 'ServiceChannel', 'name': 'wss' if url.startswith('wss') else 'ws', 'serviceUrl': url, 'additionalProperty': [_prop('rung', FLOOR_RUNG), _prop('floor', True)]}) doc: LinkedData = { '@context': CONTEXT, '@type': 'Service', 'identifier': _did(desc['pubkey']), 'serviceType': first['class'][1] if 'class' in first else 'node', 'availableChannel': channels, } if 'operator' in first: doc['provider'] = {'@type': 'Person', 'identifier': _did(first['operator'][1])} if 'expiration' in first: doc['expires'] = iso(int(first['expiration'][1])) pol = first.get('policy') if pol and len(pol) > 2 and pol[2]: doc['termsOfService'] = pol[2] extra = [_prop('capabilities', [t[1] for t in _tags(desc, 'c')])] if 'protocol' in first: extra.append(_prop('protocol', first['protocol'][1:])) if 'software' in first: extra.append(_prop('software', first['software'][1:])) if pol: extra.append(_prop('policy_sha256', pol[1])) if 'coverage' in first: extra.append(_prop('coverage', first['coverage'][1])) doc['additionalProperty'] = extra return doc
# --- standing tell -> the node, with its tier beside it ---------------------------
[docs]def standing(pubkey: str, tell: StandingTellBody | Mapping[str, Any]) -> LinkedData: """A `standing.StandingTellBody` about `pubkey`, as the thing it is about. Not a schema:Review. A tier is the outcome of this node's own windows (S0..S4 with meaning: S1 halves a limit, S4 revokes), replaceable and withdrawable; a reviewRating is an ordinal any aggregator would average. So the subject is an Organization (a node, operated, not a person) and the tell rides as PropertyValues named by fleetmesh field. The teller is `tell['target']`'s evaluator - the node serving this - and is not repeated; the HTTP origin says who is speaking. """ doc: LinkedData = { '@context': CONTEXT, '@type': 'Organization', 'identifier': _did(pubkey), 'additionalProperty': [ _prop('tier', tell['tier']), _prop('clean_windows', int(tell['clean_windows'])), _prop('evaluated_windows', int(tell['evaluated_windows'])), _prop('has_adverse_reports', bool(tell['has_adverse_reports'])), ], } return doc
# --- the pin ---------------------------------------------------------------------
[docs]class Vocabulary(TypedDict): origin: str context: str version: str types: list[str] properties: list[str]
[docs]def vocabulary() -> Vocabulary: """The pinned schema.org vocabulary, schema/schemaorg-terms.json.""" v: Vocabulary = schema_files.load_json('schemaorg-terms.json') return v
[docs]def terms(doc: Any) -> tuple[set[str], set[str]]: """(types, properties) a projection used, walked recursively. Keys with an `@` are JSON-LD keywords, not vocabulary.""" kinds: set[str] = set() props: set[str] = set() def walk(x: Any) -> None: if isinstance(x, dict): for k, v in x.items(): if k == '@type': kinds.update([v] if isinstance(v, str) else v) elif not k.startswith('@'): props.add(k) walk(v) elif isinstance(x, list): for v in x: walk(v) walk(doc) return kinds, props
[docs]def unknown_terms(doc: Any, vocab: Optional[Vocabulary] = None) -> list[str]: """Terms in `doc` the pinned vocabulary does not define. Empty is the only acceptable answer for anything a node emits.""" v = vocab or vocabulary() kinds, props = terms(doc) bad = sorted(kinds - set(v['types'])) + sorted(props - set(v['properties'])) return bad