"""Operator-linked AI/LLM inference provider adapters for FleetMesh non-deterministic compute.
Supports linking a MeshNode to external and local inference providers:
- OpenRouter (`openrouter`)
- Anthropic Claude (`claude` / `anthropic`)
- OpenAI (`openai`)
- Google Gemini (`gemini`)
- GitHub Copilot (`copilot`)
- Routstr eCash Proxy (`routstr`)
- PayPerQuery / PPQ (`payperquery` / `ppq`)
- Local vLLM / Ollama (`vllm`, `ollama`, `custom`)
Security Invariant:
All upstream URLs and credentials (`api_key`, `cashu_token`) are strictly
operator-configured on the host node. Remote compute jobs may select an
advertised `provider` alias or `model` ID, never an arbitrary URL.
"""
from __future__ import annotations
import dataclasses
import os
from typing import Any, Mapping, Optional, Sequence
[docs]@dataclasses.dataclass(frozen=True)
class ProviderPreset:
provider_id: str
label: str
endpoint: str
default_model: str
models: tuple[str, ...]
auth_scheme: str # 'none' | 'bearer' | 'anthropic' | 'cashu' | 'cashu_402'
wire_format: str # 'openai' | 'anthropic_messages'
env_key: str
rate_msat_per_token: int
extra_headers: Mapping[str, str] = dataclasses.field(default_factory=dict)
def _https(host: str, path: str = '') -> str:
return f'https://{host}{path}'
PROVIDER_PRESETS: dict[str, ProviderPreset] = {
'openrouter': ProviderPreset(
provider_id='openrouter',
label='OpenRouter (Multi-Model Aggregator)',
endpoint=_https('openrouter.ai', '/api/v1'),
default_model='anthropic/claude-3.5-sonnet',
models=(
'anthropic/claude-3.5-sonnet',
'openai/gpt-4o-mini',
'google/gemini-2.5-flash',
'meta-llama/llama-3.3-70b-instruct',
'deepseek/deepseek-r1',
),
auth_scheme='bearer',
wire_format='openai',
env_key='OPENROUTER_API_KEY',
rate_msat_per_token=15,
extra_headers={
'HTTP-Referer': _https('fleetmesh.org'),
'X-Title': 'FleetMesh NonDet Compute',
},
),
'claude': ProviderPreset(
provider_id='claude',
label='Anthropic Claude',
endpoint=_https('api.anthropic.com', '/v1'),
default_model='claude-3-7-sonnet-latest',
models=(
'claude-3-7-sonnet-latest',
'claude-3-5-sonnet-latest',
'claude-3-5-haiku-latest',
),
auth_scheme='anthropic',
wire_format='anthropic_messages',
env_key='ANTHROPIC_API_KEY',
rate_msat_per_token=18,
),
'anthropic': ProviderPreset(
provider_id='anthropic',
label='Anthropic Claude',
endpoint=_https('api.anthropic.com', '/v1'),
default_model='claude-3-7-sonnet-latest',
models=(
'claude-3-7-sonnet-latest',
'claude-3-5-sonnet-latest',
'claude-3-5-haiku-latest',
),
auth_scheme='anthropic',
wire_format='anthropic_messages',
env_key='ANTHROPIC_API_KEY',
rate_msat_per_token=18,
),
'openai': ProviderPreset(
provider_id='openai',
label='OpenAI',
endpoint=_https('api.openai.com', '/v1'),
default_model='gpt-4o-mini',
models=('gpt-4o-mini', 'gpt-4o', 'o3-mini'),
auth_scheme='bearer',
wire_format='openai',
env_key='OPENAI_API_KEY',
rate_msat_per_token=12,
),
'gemini': ProviderPreset(
provider_id='gemini',
label='Google Gemini',
endpoint=_https('generativelanguage.googleapis.com', '/v1beta/openai'),
default_model='gemini-2.5-flash',
models=('gemini-2.5-flash', 'gemini-2.5-pro', 'gemini-2.0-flash'),
auth_scheme='bearer',
wire_format='openai',
env_key='GEMINI_API_KEY',
rate_msat_per_token=10,
),
'copilot': ProviderPreset(
provider_id='copilot',
label='GitHub Copilot / GitHub Models',
endpoint=_https('models.inference.ai.azure.com'),
default_model='gpt-4o',
models=('gpt-4o', 'gpt-4o-mini', 'Llama-3.3-70B-Instruct'),
auth_scheme='bearer',
wire_format='openai',
env_key='GITHUB_TOKEN',
rate_msat_per_token=12,
),
'routstr': ProviderPreset(
provider_id='routstr',
label='Routstr (Decentralized Cashu LLM Router)',
endpoint=_https('api.routstr.com', '/v1'),
default_model='meta-llama/Llama-3.3-70B-Instruct',
models=(
'meta-llama/Llama-3.3-70B-Instruct',
'deepseek-ai/DeepSeek-R1',
'anthropic/claude-3.5-sonnet',
),
auth_scheme='cashu',
wire_format='openai',
env_key='ROUTSTR_CASHU_TOKEN',
rate_msat_per_token=12,
),
'payperquery': ProviderPreset(
provider_id='payperquery',
label='PayPerQuery / PPQ (HTTP 402 eCash Inference)',
endpoint=_https('api.ppq.ai', '/v1'),
default_model='meta-llama/Llama-3.3-70B-Instruct',
models=(
'meta-llama/Llama-3.3-70B-Instruct',
'gpt-4o-mini',
'claude-3-5-sonnet',
),
auth_scheme='cashu_402',
wire_format='openai',
env_key='PPQ_CASHU_TOKEN',
rate_msat_per_token=12,
),
'ppq': ProviderPreset(
provider_id='ppq',
label='PayPerQuery / PPQ (HTTP 402 eCash Inference)',
endpoint=_https('api.ppq.ai', '/v1'),
default_model='meta-llama/Llama-3.3-70B-Instruct',
models=(
'meta-llama/Llama-3.3-70B-Instruct',
'gpt-4o-mini',
'claude-3-5-sonnet',
),
auth_scheme='cashu_402',
wire_format='openai',
env_key='PPQ_CASHU_TOKEN',
rate_msat_per_token=12,
),
'ollama': ProviderPreset(
provider_id='ollama',
label='Ollama (Local Node)',
endpoint='http://127.0.0.1:11434/v1',
default_model='llama3.2',
models=('llama3.2', 'qwen2.5:14b', 'deepseek-r1:14b', 'mistral'),
auth_scheme='none',
wire_format='openai',
env_key='OLLAMA_API_KEY',
rate_msat_per_token=5,
),
'vllm': ProviderPreset(
provider_id='vllm',
label='vLLM (Local / Cluster GPU)',
endpoint='http://127.0.0.1:8000/v1',
default_model='meta-llama/Llama-3.3-70B-Instruct',
models=('meta-llama/Llama-3.3-70B-Instruct', 'deepseek-ai/DeepSeek-R1'),
auth_scheme='none',
wire_format='openai',
env_key='VLLM_API_KEY',
rate_msat_per_token=12,
),
}
[docs]@dataclasses.dataclass
class LinkedProviderConfig:
"""Operator-linked configuration for a single inference provider."""
provider_id: str
endpoint: str
default_model: str
models: tuple[str, ...] = ()
api_key: str = ''
cashu_token: str = ''
cashu_mint: str = ''
auth_scheme: str = 'bearer'
wire_format: str = 'openai'
rate_msat_per_token: int = 12
enabled: bool = True
label: str = ''
extra_headers: dict[str, str] = dataclasses.field(default_factory=dict)
[docs] @classmethod
def from_preset(
cls,
provider_id: str,
*,
endpoint: Optional[str] = None,
default_model: Optional[str] = None,
models: Optional[Sequence[str]] = None,
api_key: Optional[str] = None,
cashu_token: Optional[str] = None,
cashu_mint: Optional[str] = None,
rate_msat_per_token: Optional[int] = None,
enabled: bool = True,
extra_headers: Optional[Mapping[str, str]] = None,
) -> 'LinkedProviderConfig':
pid = provider_id.strip().lower()
preset = PROVIDER_PRESETS.get(pid)
if preset is None:
resolved_ep = (endpoint or 'http://127.0.0.1:8000/v1').rstrip('/')
resolved_model = default_model or 'meta-llama/Llama-3.3-70B-Instruct'
resolved_models = tuple(models) if models else (resolved_model,)
return cls(
provider_id=pid,
label=pid.upper(),
endpoint=resolved_ep,
default_model=resolved_model,
models=resolved_models,
api_key=api_key or '',
cashu_token=cashu_token or '',
cashu_mint=cashu_mint or '',
auth_scheme='bearer' if api_key else ('cashu' if cashu_token else 'none'),
wire_format='openai',
rate_msat_per_token=int(rate_msat_per_token if rate_msat_per_token is not None else 12),
enabled=enabled,
extra_headers=dict(extra_headers or {}),
)
env_secret = os.environ.get(preset.env_key, '') if preset.env_key else ''
resolved_key = api_key if api_key is not None else (env_secret if preset.auth_scheme in ('bearer', 'anthropic') else '')
resolved_cashu = cashu_token if cashu_token is not None else (env_secret if preset.auth_scheme in ('cashu', 'cashu_402') else '')
resolved_ep = (endpoint or preset.endpoint).rstrip('/')
resolved_model = default_model or preset.default_model
resolved_models = tuple(models) if models else preset.models
if resolved_model not in resolved_models:
resolved_models = (resolved_model, *resolved_models)
merged_headers = dict(preset.extra_headers)
if extra_headers:
merged_headers.update(extra_headers)
return cls(
provider_id=preset.provider_id,
label=preset.label,
endpoint=resolved_ep,
default_model=resolved_model,
models=resolved_models,
api_key=resolved_key,
cashu_token=resolved_cashu,
cashu_mint=cashu_mint or '',
auth_scheme=preset.auth_scheme,
wire_format=preset.wire_format,
rate_msat_per_token=int(rate_msat_per_token if rate_msat_per_token is not None else preset.rate_msat_per_token),
enabled=enabled,
extra_headers=merged_headers,
)
[docs] def has_credentials(self) -> bool:
if self.auth_scheme == 'none':
return True
if self.auth_scheme in ('cashu', 'cashu_402'):
return bool(self.cashu_token or self.api_key or self.cashu_mint)
return bool(self.api_key)
[docs] def is_runnable(self) -> bool:
return bool(self.enabled and self.endpoint.startswith(('http://', 'https://')) and self.has_credentials())
[docs] def parse_response(
self,
resp_data: Mapping[str, Any],
default_prompt_tokens: int = 1,
) -> tuple[str, int, int]:
if self.wire_format == 'anthropic_messages' or ('content' in resp_data and isinstance(resp_data.get('content'), list) and 'choices' not in resp_data):
blocks = resp_data.get('content') or []
text_parts: list[str] = []
for blk in blocks:
if isinstance(blk, Mapping) and blk.get('type') in ('text', None):
text_parts.append(str(blk.get('text') or ''))
raw_text = ''.join(text_parts)
usage = resp_data.get('usage') or {}
prompt_toks = int(usage.get('input_tokens', usage.get('prompt_tokens', default_prompt_tokens)))
completion_toks = int(usage.get('output_tokens', usage.get('completion_tokens', max(1, len(raw_text) // 4))))
return raw_text, prompt_toks, completion_toks
choices = resp_data.get('choices') or []
if not choices or not isinstance(choices[0], Mapping) or 'message' not in choices[0]:
raise ValueError("LLM backend returned no completion choices")
raw_text = str(choices[0]['message'].get('content') or '')
usage = resp_data.get('usage') or {}
prompt_toks = int(usage.get('prompt_tokens', default_prompt_tokens))
completion_toks = int(usage.get('completion_tokens', max(1, len(raw_text) // 4)))
return raw_text, prompt_toks, completion_toks
[docs] def to_public_dict(self) -> dict[str, Any]:
"""Serialize provider metadata without exposing secret API keys or Cashu tokens."""
return {
'provider_id': self.provider_id,
'label': self.label or self.provider_id,
'endpoint': self.endpoint,
'default_model': self.default_model,
'models': list(self.models),
'auth_scheme': self.auth_scheme,
'wire_format': self.wire_format,
'rate_msat_per_token': self.rate_msat_per_token,
'enabled': self.enabled,
'has_credentials': self.has_credentials(),
'is_runnable': self.is_runnable(),
'cashu_mint': self.cashu_mint,
}
[docs]class OperatorProviderRouter:
"""Registry of operator-linked LLM providers on a FleetMesh node."""
[docs] def __init__(
self,
providers: Optional[Sequence[LinkedProviderConfig]] = None,
default_provider: Optional[str] = None,
) -> None:
self._providers: dict[str, LinkedProviderConfig] = {}
self.default_provider: Optional[str] = default_provider
if providers:
for p in providers:
self.register_provider(p)
[docs] def register_provider(self, config: LinkedProviderConfig, set_default: bool = False) -> None:
pid = config.provider_id.strip().lower()
self._providers[pid] = config
if set_default or self.default_provider is None:
self.default_provider = pid
[docs] def remove_provider(self, provider_id: str) -> bool:
pid = provider_id.strip().lower()
existed = pid in self._providers
self._providers.pop(pid, None)
if self.default_provider == pid:
self.default_provider = next(iter(self._providers.keys()), None)
return existed
[docs] def get_provider(self, provider_id: Optional[str] = None) -> Optional[LinkedProviderConfig]:
if provider_id:
pid = provider_id.strip().lower()
if pid in self._providers:
return self._providers[pid]
if pid == 'anthropic' and 'claude' in self._providers:
return self._providers['claude']
if pid == 'claude' and 'anthropic' in self._providers:
return self._providers['anthropic']
if pid == 'ppq' and 'payperquery' in self._providers:
return self._providers['payperquery']
if pid == 'payperquery' and 'ppq' in self._providers:
return self._providers['ppq']
if self.default_provider and self.default_provider in self._providers:
return self._providers[self.default_provider]
for p in self._providers.values():
if p.is_runnable():
return p
return next(iter(self._providers.values()), None)
[docs] def resolve_for_job(self, job_payload: Mapping[str, Any]) -> Optional[LinkedProviderConfig]:
req_provider = job_payload.get('provider') or (job_payload.get('input') or {}).get('provider') if isinstance(job_payload.get('input'), Mapping) else job_payload.get('provider')
if req_provider:
found = self.get_provider(str(req_provider))
if found and found.enabled:
return found
req_model = str(job_payload.get('model') or '')
if req_model:
for p in self._providers.values():
if p.enabled and req_model in p.models:
return p
return self.get_provider()
[docs] def runnable_providers(self) -> list[LinkedProviderConfig]:
return [p for p in self._providers.values() if p.is_runnable()]
[docs] def list_public_providers(self) -> list[dict[str, Any]]:
return [p.to_public_dict() for p in self._providers.values()]
[docs] @classmethod
def from_env(cls) -> 'OperatorProviderRouter':
"""Auto-discover any operator-configured providers from environment variables."""
router = cls()
seen_Canonical: set[str] = set()
for pid, preset in PROVIDER_PRESETS.items():
if pid in ('anthropic', 'ppq'):
continue
val = os.environ.get(preset.env_key, '').strip()
if val and pid not in seen_Canonical:
seen_Canonical.add(pid)
cfg = LinkedProviderConfig.from_preset(pid)
router.register_provider(cfg)
return router