Source code for fleetmesh_node.planes.services.compute.executors.providers

"""Operator-linked AI/LLM inference provider adapters for FleetMesh non-deterministic compute.

Supports linking a MeshNode to external and local inference providers:
- OpenRouter (`openrouter`)
- Anthropic Claude (`claude` / `anthropic`)
- OpenAI (`openai`)
- Google Gemini (`gemini`)
- GitHub Copilot (`copilot`)
- Routstr eCash Proxy (`routstr`)
- PayPerQuery / PPQ (`payperquery` / `ppq`)
- Local vLLM / Ollama (`vllm`, `ollama`, `custom`)

Security Invariant:
All upstream URLs and credentials (`api_key`, `cashu_token`) are strictly
operator-configured on the host node. Remote compute jobs may select an
advertised `provider` alias or `model` ID, never an arbitrary URL.
"""
from __future__ import annotations

import dataclasses
import os
from typing import Any, Mapping, Optional, Sequence


[docs]@dataclasses.dataclass(frozen=True) class ProviderPreset: provider_id: str label: str endpoint: str default_model: str models: tuple[str, ...] auth_scheme: str # 'none' | 'bearer' | 'anthropic' | 'cashu' | 'cashu_402' wire_format: str # 'openai' | 'anthropic_messages' env_key: str rate_msat_per_token: int extra_headers: Mapping[str, str] = dataclasses.field(default_factory=dict)
def _https(host: str, path: str = '') -> str: return f'https://{host}{path}' PROVIDER_PRESETS: dict[str, ProviderPreset] = { 'openrouter': ProviderPreset( provider_id='openrouter', label='OpenRouter (Multi-Model Aggregator)', endpoint=_https('openrouter.ai', '/api/v1'), default_model='anthropic/claude-3.5-sonnet', models=( 'anthropic/claude-3.5-sonnet', 'openai/gpt-4o-mini', 'google/gemini-2.5-flash', 'meta-llama/llama-3.3-70b-instruct', 'deepseek/deepseek-r1', ), auth_scheme='bearer', wire_format='openai', env_key='OPENROUTER_API_KEY', rate_msat_per_token=15, extra_headers={ 'HTTP-Referer': _https('fleetmesh.org'), 'X-Title': 'FleetMesh NonDet Compute', }, ), 'claude': ProviderPreset( provider_id='claude', label='Anthropic Claude', endpoint=_https('api.anthropic.com', '/v1'), default_model='claude-3-7-sonnet-latest', models=( 'claude-3-7-sonnet-latest', 'claude-3-5-sonnet-latest', 'claude-3-5-haiku-latest', ), auth_scheme='anthropic', wire_format='anthropic_messages', env_key='ANTHROPIC_API_KEY', rate_msat_per_token=18, ), 'anthropic': ProviderPreset( provider_id='anthropic', label='Anthropic Claude', endpoint=_https('api.anthropic.com', '/v1'), default_model='claude-3-7-sonnet-latest', models=( 'claude-3-7-sonnet-latest', 'claude-3-5-sonnet-latest', 'claude-3-5-haiku-latest', ), auth_scheme='anthropic', wire_format='anthropic_messages', env_key='ANTHROPIC_API_KEY', rate_msat_per_token=18, ), 'openai': ProviderPreset( provider_id='openai', label='OpenAI', endpoint=_https('api.openai.com', '/v1'), default_model='gpt-4o-mini', models=('gpt-4o-mini', 'gpt-4o', 'o3-mini'), auth_scheme='bearer', wire_format='openai', env_key='OPENAI_API_KEY', rate_msat_per_token=12, ), 'gemini': ProviderPreset( provider_id='gemini', label='Google Gemini', endpoint=_https('generativelanguage.googleapis.com', '/v1beta/openai'), default_model='gemini-2.5-flash', models=('gemini-2.5-flash', 'gemini-2.5-pro', 'gemini-2.0-flash'), auth_scheme='bearer', wire_format='openai', env_key='GEMINI_API_KEY', rate_msat_per_token=10, ), 'copilot': ProviderPreset( provider_id='copilot', label='GitHub Copilot / GitHub Models', endpoint=_https('models.inference.ai.azure.com'), default_model='gpt-4o', models=('gpt-4o', 'gpt-4o-mini', 'Llama-3.3-70B-Instruct'), auth_scheme='bearer', wire_format='openai', env_key='GITHUB_TOKEN', rate_msat_per_token=12, ), 'routstr': ProviderPreset( provider_id='routstr', label='Routstr (Decentralized Cashu LLM Router)', endpoint=_https('api.routstr.com', '/v1'), default_model='meta-llama/Llama-3.3-70B-Instruct', models=( 'meta-llama/Llama-3.3-70B-Instruct', 'deepseek-ai/DeepSeek-R1', 'anthropic/claude-3.5-sonnet', ), auth_scheme='cashu', wire_format='openai', env_key='ROUTSTR_CASHU_TOKEN', rate_msat_per_token=12, ), 'payperquery': ProviderPreset( provider_id='payperquery', label='PayPerQuery / PPQ (HTTP 402 eCash Inference)', endpoint=_https('api.ppq.ai', '/v1'), default_model='meta-llama/Llama-3.3-70B-Instruct', models=( 'meta-llama/Llama-3.3-70B-Instruct', 'gpt-4o-mini', 'claude-3-5-sonnet', ), auth_scheme='cashu_402', wire_format='openai', env_key='PPQ_CASHU_TOKEN', rate_msat_per_token=12, ), 'ppq': ProviderPreset( provider_id='ppq', label='PayPerQuery / PPQ (HTTP 402 eCash Inference)', endpoint=_https('api.ppq.ai', '/v1'), default_model='meta-llama/Llama-3.3-70B-Instruct', models=( 'meta-llama/Llama-3.3-70B-Instruct', 'gpt-4o-mini', 'claude-3-5-sonnet', ), auth_scheme='cashu_402', wire_format='openai', env_key='PPQ_CASHU_TOKEN', rate_msat_per_token=12, ), 'ollama': ProviderPreset( provider_id='ollama', label='Ollama (Local Node)', endpoint='http://127.0.0.1:11434/v1', default_model='llama3.2', models=('llama3.2', 'qwen2.5:14b', 'deepseek-r1:14b', 'mistral'), auth_scheme='none', wire_format='openai', env_key='OLLAMA_API_KEY', rate_msat_per_token=5, ), 'vllm': ProviderPreset( provider_id='vllm', label='vLLM (Local / Cluster GPU)', endpoint='http://127.0.0.1:8000/v1', default_model='meta-llama/Llama-3.3-70B-Instruct', models=('meta-llama/Llama-3.3-70B-Instruct', 'deepseek-ai/DeepSeek-R1'), auth_scheme='none', wire_format='openai', env_key='VLLM_API_KEY', rate_msat_per_token=12, ), }
[docs]@dataclasses.dataclass class LinkedProviderConfig: """Operator-linked configuration for a single inference provider.""" provider_id: str endpoint: str default_model: str models: tuple[str, ...] = () api_key: str = '' cashu_token: str = '' cashu_mint: str = '' auth_scheme: str = 'bearer' wire_format: str = 'openai' rate_msat_per_token: int = 12 enabled: bool = True label: str = '' extra_headers: dict[str, str] = dataclasses.field(default_factory=dict)
[docs] @classmethod def from_preset( cls, provider_id: str, *, endpoint: Optional[str] = None, default_model: Optional[str] = None, models: Optional[Sequence[str]] = None, api_key: Optional[str] = None, cashu_token: Optional[str] = None, cashu_mint: Optional[str] = None, rate_msat_per_token: Optional[int] = None, enabled: bool = True, extra_headers: Optional[Mapping[str, str]] = None, ) -> 'LinkedProviderConfig': pid = provider_id.strip().lower() preset = PROVIDER_PRESETS.get(pid) if preset is None: resolved_ep = (endpoint or 'http://127.0.0.1:8000/v1').rstrip('/') resolved_model = default_model or 'meta-llama/Llama-3.3-70B-Instruct' resolved_models = tuple(models) if models else (resolved_model,) return cls( provider_id=pid, label=pid.upper(), endpoint=resolved_ep, default_model=resolved_model, models=resolved_models, api_key=api_key or '', cashu_token=cashu_token or '', cashu_mint=cashu_mint or '', auth_scheme='bearer' if api_key else ('cashu' if cashu_token else 'none'), wire_format='openai', rate_msat_per_token=int(rate_msat_per_token if rate_msat_per_token is not None else 12), enabled=enabled, extra_headers=dict(extra_headers or {}), ) env_secret = os.environ.get(preset.env_key, '') if preset.env_key else '' resolved_key = api_key if api_key is not None else (env_secret if preset.auth_scheme in ('bearer', 'anthropic') else '') resolved_cashu = cashu_token if cashu_token is not None else (env_secret if preset.auth_scheme in ('cashu', 'cashu_402') else '') resolved_ep = (endpoint or preset.endpoint).rstrip('/') resolved_model = default_model or preset.default_model resolved_models = tuple(models) if models else preset.models if resolved_model not in resolved_models: resolved_models = (resolved_model, *resolved_models) merged_headers = dict(preset.extra_headers) if extra_headers: merged_headers.update(extra_headers) return cls( provider_id=preset.provider_id, label=preset.label, endpoint=resolved_ep, default_model=resolved_model, models=resolved_models, api_key=resolved_key, cashu_token=resolved_cashu, cashu_mint=cashu_mint or '', auth_scheme=preset.auth_scheme, wire_format=preset.wire_format, rate_msat_per_token=int(rate_msat_per_token if rate_msat_per_token is not None else preset.rate_msat_per_token), enabled=enabled, extra_headers=merged_headers, )
[docs] def has_credentials(self) -> bool: if self.auth_scheme == 'none': return True if self.auth_scheme in ('cashu', 'cashu_402'): return bool(self.cashu_token or self.api_key or self.cashu_mint) return bool(self.api_key)
[docs] def is_runnable(self) -> bool: return bool(self.enabled and self.endpoint.startswith(('http://', 'https://')) and self.has_credentials())
[docs] def build_headers(self, per_query_cashu_token: Optional[str] = None) -> dict[str, str]: headers: dict[str, str] = { 'Content-Type': 'application/json', 'User-Agent': 'fleetmesh-llm/1.1', } headers.update(self.extra_headers) active_cashu = per_query_cashu_token or self.cashu_token if self.auth_scheme == 'anthropic': if self.api_key: headers['x-api-key'] = self.api_key headers.setdefault('anthropic-version', '2023-06-01') elif self.auth_scheme in ('cashu', 'cashu_402'): if active_cashu: headers['X-Cashu'] = active_cashu headers['Authorization'] = f'Bearer {active_cashu}' elif self.api_key: headers['Authorization'] = f'Bearer {self.api_key}' elif self.auth_scheme == 'bearer' or self.api_key: if self.api_key: headers['Authorization'] = f'Bearer {self.api_key}' return headers
[docs] def format_request( self, *, messages: Sequence[Mapping[str, Any]], model: Optional[str] = None, max_tokens: int = 512, temperature: float = 0.7, top_p: float = 1.0, seed: Optional[int] = None, grammar: Optional[Mapping[str, Any]] = None, stream: bool = False, ) -> tuple[str, dict[str, Any]]: target_model = model or self.default_model if self.wire_format == 'anthropic_messages': url = f"{self.endpoint}/messages" if not self.endpoint.endswith('/messages') else self.endpoint system_parts: list[str] = [] chat_msgs: list[dict[str, str]] = [] for m in messages: role = str(m.get('role', 'user')) content = str(m.get('content', '')) if role == 'system': if content: system_parts.append(content) else: chat_msgs.append({ 'role': 'assistant' if role == 'assistant' else 'user', 'content': content or ' ', }) if not chat_msgs: chat_msgs = [{'role': 'user', 'content': 'Hello'}] body: dict[str, Any] = { 'model': target_model, 'messages': chat_msgs, 'max_tokens': max_tokens, 'temperature': temperature, 'stream': stream, } if system_parts: body['system'] = '\n\n'.join(system_parts) return url, body url = f"{self.endpoint}/chat/completions" if not self.endpoint.endswith('/chat/completions') else self.endpoint req_body: dict[str, Any] = { 'model': target_model, 'messages': [dict(m) for m in messages], 'max_tokens': max_tokens, 'temperature': temperature, 'top_p': top_p, 'stream': stream, } if seed is not None: req_body['seed'] = int(seed) if grammar: g_type = grammar.get('type') if g_type == 'json_schema' or 'json_schema' in grammar or grammar.get('response_format') == 'json_object': req_body['response_format'] = { 'type': 'json_object', 'schema': grammar.get('definition') or grammar.get('json_schema') or grammar.get('schema_cid'), } elif g_type == 'regex' or 'guided_regex' in grammar: req_body['guided_regex'] = grammar.get('definition') or grammar.get('guided_regex') return url, req_body
[docs] def parse_response( self, resp_data: Mapping[str, Any], default_prompt_tokens: int = 1, ) -> tuple[str, int, int]: if self.wire_format == 'anthropic_messages' or ('content' in resp_data and isinstance(resp_data.get('content'), list) and 'choices' not in resp_data): blocks = resp_data.get('content') or [] text_parts: list[str] = [] for blk in blocks: if isinstance(blk, Mapping) and blk.get('type') in ('text', None): text_parts.append(str(blk.get('text') or '')) raw_text = ''.join(text_parts) usage = resp_data.get('usage') or {} prompt_toks = int(usage.get('input_tokens', usage.get('prompt_tokens', default_prompt_tokens))) completion_toks = int(usage.get('output_tokens', usage.get('completion_tokens', max(1, len(raw_text) // 4)))) return raw_text, prompt_toks, completion_toks choices = resp_data.get('choices') or [] if not choices or not isinstance(choices[0], Mapping) or 'message' not in choices[0]: raise ValueError("LLM backend returned no completion choices") raw_text = str(choices[0]['message'].get('content') or '') usage = resp_data.get('usage') or {} prompt_toks = int(usage.get('prompt_tokens', default_prompt_tokens)) completion_toks = int(usage.get('completion_tokens', max(1, len(raw_text) // 4))) return raw_text, prompt_toks, completion_toks
[docs] def to_public_dict(self) -> dict[str, Any]: """Serialize provider metadata without exposing secret API keys or Cashu tokens.""" return { 'provider_id': self.provider_id, 'label': self.label or self.provider_id, 'endpoint': self.endpoint, 'default_model': self.default_model, 'models': list(self.models), 'auth_scheme': self.auth_scheme, 'wire_format': self.wire_format, 'rate_msat_per_token': self.rate_msat_per_token, 'enabled': self.enabled, 'has_credentials': self.has_credentials(), 'is_runnable': self.is_runnable(), 'cashu_mint': self.cashu_mint, }
[docs]class OperatorProviderRouter: """Registry of operator-linked LLM providers on a FleetMesh node."""
[docs] def __init__( self, providers: Optional[Sequence[LinkedProviderConfig]] = None, default_provider: Optional[str] = None, ) -> None: self._providers: dict[str, LinkedProviderConfig] = {} self.default_provider: Optional[str] = default_provider if providers: for p in providers: self.register_provider(p)
[docs] def register_provider(self, config: LinkedProviderConfig, set_default: bool = False) -> None: pid = config.provider_id.strip().lower() self._providers[pid] = config if set_default or self.default_provider is None: self.default_provider = pid
[docs] def remove_provider(self, provider_id: str) -> bool: pid = provider_id.strip().lower() existed = pid in self._providers self._providers.pop(pid, None) if self.default_provider == pid: self.default_provider = next(iter(self._providers.keys()), None) return existed
[docs] def get_provider(self, provider_id: Optional[str] = None) -> Optional[LinkedProviderConfig]: if provider_id: pid = provider_id.strip().lower() if pid in self._providers: return self._providers[pid] if pid == 'anthropic' and 'claude' in self._providers: return self._providers['claude'] if pid == 'claude' and 'anthropic' in self._providers: return self._providers['anthropic'] if pid == 'ppq' and 'payperquery' in self._providers: return self._providers['payperquery'] if pid == 'payperquery' and 'ppq' in self._providers: return self._providers['ppq'] if self.default_provider and self.default_provider in self._providers: return self._providers[self.default_provider] for p in self._providers.values(): if p.is_runnable(): return p return next(iter(self._providers.values()), None)
[docs] def resolve_for_job(self, job_payload: Mapping[str, Any]) -> Optional[LinkedProviderConfig]: req_provider = job_payload.get('provider') or (job_payload.get('input') or {}).get('provider') if isinstance(job_payload.get('input'), Mapping) else job_payload.get('provider') if req_provider: found = self.get_provider(str(req_provider)) if found and found.enabled: return found req_model = str(job_payload.get('model') or '') if req_model: for p in self._providers.values(): if p.enabled and req_model in p.models: return p return self.get_provider()
[docs] def runnable_providers(self) -> list[LinkedProviderConfig]: return [p for p in self._providers.values() if p.is_runnable()]
[docs] def list_public_providers(self) -> list[dict[str, Any]]: return [p.to_public_dict() for p in self._providers.values()]
[docs] @classmethod def from_env(cls) -> 'OperatorProviderRouter': """Auto-discover any operator-configured providers from environment variables.""" router = cls() seen_Canonical: set[str] = set() for pid, preset in PROVIDER_PRESETS.items(): if pid in ('anthropic', 'ppq'): continue val = os.environ.get(preset.env_key, '').strip() if val and pid not in seen_Canonical: seen_Canonical.add(pid) cfg = LinkedProviderConfig.from_preset(pid) router.register_provider(cfg) return router