"""The unplug: a process that cannot reach a network, and a scan for what it might reach for.
The specification makes one claim about dependence and repeats it in three
places. No bootstrap host, DNS domain or single operator is a hardwired
constant. relay.example.org is an example and nothing else. Losing every seed
domain slows a first run and does nothing to an established node. ยง Conformance
turns that into a test: the whole suite again with every default anchor deleted,
every fleetmesh.org lookup failing, and the bootstrap list stripped.
A claim like that decays quietly. Nobody adds a dependency on purpose; somebody
adds a convenience, and a year later the network has a single point of failure
nobody chose. So this is two mechanisms, not one:
cut_the_network() makes any socket or name lookup raise. Run a suite
under it and a suite that reaches out cannot pass.
reachable_constants() reads the source for addresses. A name nothing
resolves is still a dependency the day someone writes
the code that resolves it.
The second needs a rule for what an address *is*, because this protocol is full
of https:// strings that are names rather than locations - the DIDComm protocol
identifiers, which the specification says are never dereferenced. The rule:
an example host example.org and friends. Fine, and the only host a
document is allowed to write down.
a loopback address 127.0.0.1 or localhost. Nothing to unplug: it
never leaves the machine.
a host with no dot wss://one. A placeholder nothing could resolve.
a declared identifier a protocol URI schema/messages.json names, or a
didcomm.org protocol. Never fetched.
anything else reported. Including our own domain: a URL under
fleetmesh.org that is not one of the identifiers is
something a node would fetch, and the unplug test
deletes it.
Standard library only.
"""
from __future__ import annotations
import json
import os
import re
import socket
from typing import Any, Callable, Optional, Sequence
from ..conformance.errors import FleetmeshError
EXAMPLE_HOSTS: tuple[str, ...] = ('example.org', 'example.com', 'example.net', 'example.invalid')
LOOPBACK: tuple[str, ...] = ('127.0.0.1', 'localhost', '::1', '[::1]')
IDENTIFIER_HOSTS: tuple[str, ...] = ('didcomm.org', 'w3id.org', 'www.w3.org')
URL: re.Pattern[str] = re.compile(r'(?:https?|wss?)://[A-Za-z0-9._~:/?#@!$&*+,;=%-]+')
SCANNED: tuple[str, ...] = ('fleetmesh_core', 'schema', 'manifests')
[docs]class Reached(FleetmeshError, RuntimeError):
"""Something tried to open a socket or resolve a name. Under the unplug
test that is the failure, not an error on the way to one."""
[docs]def cut_the_network() -> Callable[..., Any]:
"""Make the network unavailable to this process, loudly.
Not a firewall and not a sandbox: it replaces the four entry points in
socket that everything else in the standard library goes through, so a
urllib call, an http.client call or a raw connect all end in Reached with
the thing they were reaching for in the message.
"""
def refuse(*args: Any, **kwargs: Any) -> Any:
target = args[0] if args else ''
raise Reached('the unplug test forbids reaching the network%s'
% (': %r' % (target,) if target else ''))
# Rebinding module attributes on a stdlib module is exactly what the
# typeshed stub says one must not do (socket.socket is a class there), so
# this goes through setattr: the same runtime effect, and honest about
# being outside what the stub models.
__import__('ssl')
for name in ('socket', 'create_connection', 'getaddrinfo', 'gethostbyname', 'gethostbyname_ex'):
setattr(socket, name, refuse)
return refuse
[docs]def declared_identifiers(root: str) -> set[str]:
"""Protocol URIs that name a protocol rather than a location.
Read from schema/messages.json rather than listed here, so a protocol added
to the schema does not have to be added twice, and a URL that is *not* in
the schema cannot pass as an identifier by looking like one.
"""
path = os.path.join(root, 'schema', 'messages.json')
with open(path, encoding='utf-8') as fh:
doc = json.load(fh)
ids = set(doc.get('protocols', {})) | set(doc.get('reused_protocols', []))
# The DID document's @context values are JSON-LD identifiers a node never
# fetches; constants.json lists them under descriptor.did_contexts.
with open(os.path.join(root, 'schema', 'constants.json'), encoding='utf-8') as fh:
constants = json.load(fh)
ids |= set(constants.get('descriptor', {}).get('did_contexts', []))
# Capability URIs (constants.json capability.vocabulary) are names a kind
# 30832 grant carries under `can`, in the same form as the protocol
# identifiers above: compared, never fetched.
prefix = constants.get('capability', {}).get('uri_prefix')
if prefix:
ids.add(prefix)
# The schema.org vocabulary pin names the @context linked_data emits. A
# JSON-LD context identifier like the DID ones above: never fetched. The
# pin is outside the protocol digest, so the context lives there and not
# in constants.json.
pin = os.path.join(root, 'schema', 'schemaorg-terms.json')
if os.path.exists(pin):
with open(pin, encoding='utf-8') as fh:
ctx = json.load(fh).get('context')
if isinstance(ctx, str):
ids.add(ctx)
# Formal schemas for DIDComm messages declare message type and protocol URIs
for schema_dir in (os.path.join(root, 'schemas', 'didcomm'),
os.path.join(root, 'packages', 'fleetmesh-schema', 'schemas', 'didcomm')):
if os.path.isdir(schema_dir):
for dirpath, _, filenames in os.walk(schema_dir):
for name in filenames:
if name.endswith('.json'):
spath = os.path.join(dirpath, name)
try:
with open(spath, encoding='utf-8') as sf:
sdata = json.load(sf)
except (OSError, json.JSONDecodeError) as e:
# A schema we cannot read is a schema whose identifiers
# we cannot allow-list; silently skipping it would
# turn every URL it declares into a false positive.
raise RuntimeError('cannot read schema file %s: %s' % (spath, e)) from e
if not isinstance(sdata, dict):
continue
const_t = sdata.get('properties', {}).get('type', {}).get('const')
if const_t and isinstance(const_t, str):
ids.add(const_t)
if '/' in const_t:
ids.add(const_t.rsplit('/', 1)[0])
return ids
def _host(url: str) -> str:
rest = url.split('://', 1)[1]
return rest.split('/', 1)[0].split('?', 1)[0].split('#', 1)[0].split(':', 1)[0]
[docs]def classify(url: str, identifiers: set[str] | Sequence[str]) -> Optional[str]:
"""Why this URL is allowed to be here, or None if it is not."""
url = url.rstrip('\'",.);')
host = _host(url)
if host in EXAMPLE_HOSTS or any(host.endswith('.' + h) for h in EXAMPLE_HOSTS):
return 'an example host'
if host in LOOPBACK:
return 'a loopback address, which never leaves the machine'
if '.' not in host:
return 'a placeholder, not a resolvable name'
if url in identifiers or any(url.startswith(i) for i in identifiers):
return 'a protocol identifier schema/messages.json declares'
if host in IDENTIFIER_HOSTS:
return 'a registered identifier namespace, never dereferenced'
return None
[docs]def reachable_constants(root: str, scanned: Sequence[str] = SCANNED) -> list[tuple[str, str, str]]:
"""Every address in the shipped source that a node could actually go to.
Returns [(relative path, url, host)]. Empty is the claim: the reference,
the digested constants and the example manifests name no host anyone has
to reach.
test_*.py is skipped. A test is not something a node runs, and this suite
has to be able to write down the URLs it rejects in order to check that it
rejects them. The suites are covered by the other half of this module
instead: they run with the network cut, so a test that reaches out fails
rather than being scanned for.
"""
identifiers = declared_identifiers(root)
out: list[tuple[str, str, str]] = []
for folder in scanned:
base = os.path.join(root, folder)
for dirpath, dirnames, filenames in os.walk(base):
dirnames[:] = [d for d in dirnames if d != '__pycache__']
for name in sorted(filenames):
if not name.endswith(('.py', '.json', '.yaml', '.yml', '.toml')):
continue
if name.startswith('test_'):
continue
path = os.path.join(dirpath, name)
with open(path, encoding='utf-8') as fh:
text = fh.read()
for url in sorted(set(URL.findall(text))):
if classify(url, identifiers) is None:
out.append((os.path.relpath(path, root),
url.rstrip('\'",.);'), _host(url)))
return sorted(set(out))