From 3c2f8c57ca46bd08d111aa64111f75bd6b052363 Mon Sep 17 00:00:00 2001 From: Robert Helewka Date: Sun, 24 May 2026 23:52:53 -0400 Subject: [PATCH] feat(observability): add SearXNG, Argos, and Pallas monitoring - Add SearXNG syslog ingestion and blackbox health probes on miranda and rosalind for per-host attributable failure detection - Scrape Argos MCP application metrics from miranda - Add Pallas dashboard panels for downstream availability and turn error ratios --- ansible/alloy/miranda/config.alloy.j2 | 53 + ansible/alloy/rosalind/config.alloy.j2 | 25 + ansible/inventory/group_vars/all/vars.yml | 19 +- ansible/inventory/group_vars/all/vault.yml | 945 +++++++++--------- ansible/inventory/host_vars/ariel.incus.yml | 3 +- ansible/inventory/host_vars/miranda.incus.yml | 37 +- .../inventory/host_vars/prospero.incus.yml | 33 +- ansible/inventory/host_vars/puck.incus.yml | 17 +- ansible/inventory/host_vars/titania.incus.yml | 8 +- ansible/inventory/host_vars/umbriel.incus.yml | 3 +- ansible/mcpo/config.json.j2 | 30 - ansible/neo4j/docker-compose.yml.j2 | 1 + ansible/neo4j_mcp/docker-compose.yml.j2 | 8 +- ansible/pplg/dashboards_provider.yml.j2 | 23 - ansible/pplg/deploy.yml | 34 - ansible/pplg/prometheus.yml.j2 | 44 +- ansible/pplg/puck_services_dashboard.json.j2 | 242 ----- ansible/searxng/deploy.yml | 75 -- ansible/searxng/deploy_oauth2.yml | 86 ++ ansible/site.yml | 3 + dashboards/argos.json | 307 ++++++ dashboards/daedalus_stack.json | 702 +++++++++++++ dashboards/searxng.json | 202 ++++ docs/ouranos.md | 6 +- 24 files changed, 1968 insertions(+), 938 deletions(-) delete mode 100644 ansible/pplg/dashboards_provider.yml.j2 delete mode 100644 ansible/pplg/puck_services_dashboard.json.j2 create mode 100644 ansible/searxng/deploy_oauth2.yml create mode 100644 dashboards/argos.json create mode 100644 dashboards/daedalus_stack.json create mode 100644 dashboards/searxng.json diff --git a/ansible/alloy/miranda/config.alloy.j2 b/ansible/alloy/miranda/config.alloy.j2 index 72d0830..778c6e0 100644 --- a/ansible/alloy/miranda/config.alloy.j2 +++ b/ansible/alloy/miranda/config.alloy.j2 @@ -93,6 +93,20 @@ loki.source.syslog "gitea_mcp_logs" { forward_to = [loki.write.default.receiver] } +loki.source.syslog "searxng_logs" { + listener { + address = "127.0.0.1:{{searxng_syslog_port}}" + protocol = "tcp" + syslog_format = "{{ syslog_format }}" + labels = { + job = "searxng", + hostname = "{{inventory_hostname}}", + environment = "{{deployment_environment}}", + } + } + forward_to = [loki.write.default.receiver] +} + prometheus.exporter.unix "default" { include_exporter_metrics = true disable_collectors = ["mdadm"] @@ -104,6 +118,45 @@ prometheus.scrape "default" { job_name = "mcp_docker_host" } +// Argos MCP application metrics (/metrics is exposed by argos itself; see +// argos/argos_searxng/metrics.py). +prometheus.scrape "argos" { + targets = [{ + __address__ = "127.0.0.1:{{argos_port}}", + job = "argos", + instance = "{{inventory_hostname}}", + hostname = "{{inventory_hostname}}", + environment = "{{deployment_environment}}", + }] + forward_to = [prometheus.remote_write.default.receiver] + scrape_interval = "30s" + metrics_path = "/metrics" +} + +// Independent verification that this host's SearXNG instance answers /healthz +// (Argos's own per-instance gauge can lie — argos itself could be sick). +prometheus.exporter.blackbox "searxng" { + config = "{ modules: { http_2xx: { prober: http, timeout: 5s, http: { valid_status_codes: [200] } } } }" + + target { + name = "{{inventory_hostname}}" + address = "http://127.0.0.1:{{searxng_port}}/healthz" + module = "http_2xx" + labels = { + service = "searxng", + hostname = "{{inventory_hostname}}", + environment = "{{deployment_environment}}", + } + } +} + +prometheus.scrape "searxng_blackbox" { + targets = prometheus.exporter.blackbox.searxng.targets + forward_to = [prometheus.remote_write.default.receiver] + scrape_interval = "30s" + job_name = "searxng_blackbox" +} + prometheus.remote_write "default" { endpoint { url = "{{prometheus_remote_write_url}}" diff --git a/ansible/alloy/rosalind/config.alloy.j2 b/ansible/alloy/rosalind/config.alloy.j2 index 3841c86..5c6c1f2 100644 --- a/ansible/alloy/rosalind/config.alloy.j2 +++ b/ansible/alloy/rosalind/config.alloy.j2 @@ -190,6 +190,31 @@ prometheus.scrape "gitea" { bearer_token = "{{gitea_metrics_token}}" } +// Independent verification that this host's SearXNG instance answers /healthz. +// Argos (on miranda) load-balances across this instance and miranda's own; +// each host's Alloy probes its local SearXNG so failures are attributable. +prometheus.exporter.blackbox "searxng" { + config = "{ modules: { http_2xx: { prober: http, timeout: 5s, http: { valid_status_codes: [200] } } } }" + + target { + name = "{{inventory_hostname}}" + address = "http://127.0.0.1:{{searxng_port}}/healthz" + module = "http_2xx" + labels = { + service = "searxng", + hostname = "{{inventory_hostname}}", + environment = "{{deployment_environment}}", + } + } +} + +prometheus.scrape "searxng_blackbox" { + targets = prometheus.exporter.blackbox.searxng.targets + forward_to = [prometheus.remote_write.default.receiver] + scrape_interval = "30s" + job_name = "searxng_blackbox" +} + // Prometheus remote write endpoint prometheus.remote_write "default" { endpoint { diff --git a/ansible/inventory/group_vars/all/vars.yml b/ansible/inventory/group_vars/all/vars.yml index f56bdbb..d242ec8 100644 --- a/ansible/inventory/group_vars/all/vars.yml +++ b/ansible/inventory/group_vars/all/vars.yml @@ -46,11 +46,11 @@ freecad_mcp_git_ref: "main" # Docker image versions (third-party) # Centralized for vulnerability tracking and controlled upgrades casdoor_image_version: "3.0.1" -flower_image_version: "latest" -grafana_mcp_image_version: "latest" +flower_image_version: latest +grafana_mcp_image_version: latest gitea_mcp_image_version: latest -neo4j_image_version: community-trixie -neo4j_mcp_image_version: "latest" +neo4j_version: latest +neo4j_mcp_image_version: latest memcached_image_version: "1.6-trixie" nginx_image_version: "1.27-bookworm" nginx_exporter_image_version: "1.4" @@ -59,18 +59,17 @@ rabbitmq_image_version: "3-management-alpine" searxng_image_version: "latest" # MCP URLs -argos_mcp_url: http://miranda.incus:25534/mcp +argos_mcp_url: http://miranda.incus:20861/mcp angelia_mcp_url: https://ouranos.helu.ca/mcp/ angelia_mcp_auth: "{{ vault_angelia_mcp_auth }}" caliban_mcp_url: http://caliban.incus:22021/mcp -gitea_mcp_url: http://miranda.incus:25535/mcp +gitea_mcp_url: http://miranda.incus:22062/mcp gitea_mcp_access_token: "{{ vault_gitea_mcp_access_token }}" github_personal_access_token: "{{ vault_github_personal_access_token }}" -grafana_mcp_url: http://miranda.incus:25533/mcp +grafana_mcp_url: http://miranda.incus:22063/mcp huggingface_mcp_token: "{{ vault_huggingface_mcp_token }}" -neo4j_mcp_url: http://circe.helu.ca:22034/mcp -nike_mcp_url: http://puck.incus:22031/mcp -korax_mcp_url: http://korax.helu.ca:22021/mcp +neo4j_mcp_url: http://miranda.incus:22064/mcp +nike_mcp_url: http://puck.incus:20661/mcp rommie_mcp_url: https://rommie.ouranos.helu.ca/mcp freecad_mcp_url: https://freecad-mcp.ouranos.helu.ca/mcp diff --git a/ansible/inventory/group_vars/all/vault.yml b/ansible/inventory/group_vars/all/vault.yml index 9acf8a6..710cb5b 100644 --- a/ansible/inventory/group_vars/all/vault.yml +++ b/ansible/inventory/group_vars/all/vault.yml @@ -1,473 +1,474 @@ $ANSIBLE_VAULT;1.1;AES256 -38333666356533343964653461626466306566343366376231313939313562306532333261633064 -6262333836396435626532333333653934393030646335390a383963356365336537633730396631 -34346338663365326664626530396636616139646431323263386364653162626465303537343930 -3934376238643535300a386364613538363436636564306565353133396235303563643331393738 -62653633633563633333393664373663363536393063376266646530346537663765646639333732 -34393963616162333834643461396466396336313865323030633838313034396637373834636464 -39376261346238383862303565323064616430376639343766356566363039383764646436656330 -33313831636631393931353335353266363831623039326433636134623931383634326466383332 -38316366653835383635646163623638636630633261303030373131666336626562653861626230 -64306564666363643734643462373166663237623035346335626131353036366537343061613134 -31636132363662376265356166626165633738653632653362363463306438646635623633366433 -65623431393335313963653466626261663731626235346230613531383064333833386638313436 -65653736646630643864393730653863306337656236396164613266353162623832383637663135 -34373365656437653534343561643761623865656332363533643132316639333463656535333431 -30656435633165653432363536623737663861313364346239636332393761623162313032613139 -61383735333033373963396333303062393966393434633462653833343061323234303438383664 -64323765363235376566356563366230616263356465333235663663643531633032633332646161 -33363862643765343966633061663766393036663539663035306537333539376333636336666664 -39366435366135643766646233366431383532613138366162656433343737636436663561613536 -30643735656537373838376232393333373830306131393832383136623039383836323937373561 -36643761623438316637636539393731343436333464383864623335343365616461393561396339 -38656538366166656337613661636537633666656532373534373061363463376631353835633334 -31306533353039386533613030316431633962323939623633616136353266313764643662393866 -32326565653261633235306262356139616439313930343864316563616336323130316332386333 -37396437353766323461623234343138643265646532636235316166376466336463623834613732 -34396264303132626134643934373637636132643362363434343866656631666562353834323630 -30653839363234323062643736363932323032376262363164653566363862346439663130393663 -37323761373762623362336237366237383066336330343338666330373637363730373966326463 -37363834316433616334616337633435396363633164656466326130313134356439326136393332 -39353265616131386538363162613336396137353939643130383766353739353030373536653861 -34393333333637393465613130313237626135376439313166323433363230353235653536653261 -32393736346462633530336266323964376435323030376363623134396139663630613262313531 -65633930653537313634316361383234616135646132393261316364303733333663326139646431 -61653765336535373635393563333566333338336636393236633430643164646134316332336163 -39663262383332303064636565613139303837643437356536663439653566356365373766373138 -35393733616664396166323932316464336434353564376635356136396330313030656266646463 -62366630383664613432326637316236643665353431333065336533306337643936303763313861 -38623236613231633131343664376533373534343662383635303738616335353935373932316662 -32346433323530653035353262633065623735636364613238343039633961653963343135666136 -63323162643137396265373732396565376531663037303838383834653665613233376237363664 -63373563616132313961373230323161383732376364303139653662663064373237626534633163 -34336235363639356665346536373133656664323836383235623535316232386137616466386561 -38363961646265393337616666643863366634663365613833666136346232346431396430666162 -38336431333462623739616334366534656165636138386333316636303263376434623831393563 -64373233643839353666373235316637646436356362303435393835666236623235613436663938 -37316664633866336232313161626462323039333234316564366534656330613035363236616666 -32623137303764323535633866383430303461336561653132663434343334636338383361383964 -65656535653362323839346538613561373733613135383239333037623163343433336633356333 -63663839386165346161313130323733643236613531343165336434353663306264303830643938 -35376536626437326137653636653763356636386237303830376538313661346136626634353432 -61396330646133663438323039376336386463643564313036373936356435653535623139663765 -63653564656263303738313939373933646339656634623134663437316236646464373233643937 -31653261313530646239636530376533613535373830373964343939393033343665393030643437 -38616137626463336562353261316133613534663364663230356130616634626464323333376539 -66326234323462393261373664316132343666616534333635656632323133626264643465346536 -32306566383935353930303931633531636564383233336539363933396464303062613937393932 -65336331376266316230333261633732383834346233646166303230376337303437663665326261 -36626439353466343834343138626432356239303661316630326535323131306632613830616363 -30373730616636643435353863613965383336623563393935656436313234303162386235363565 -63336462663663393939303131643936653437613861396234323235646135363339386434653764 -30326337343963346535353262656632393831383037323931653765303439376464316236376162 -34646266376265646531316133633330663863376565316231646439646636326535623664616163 -38653236353335316132623166366162623934623562313035663035343065343936316661343033 -38356363623536373439646664643065633136306334373231643362383636646265363030356638 -30366639303763616665623737653331666537303334393934346431333139393231393738316330 -39623166363763646166323566313761353161666635663439653133343639326266333364623037 -61323038303463363865633532336535383638663332616634663266623664353630663061333037 -61396566663135313131316462653839623366363361663032353634623937383863653230393739 -30333462663936643461333334633666386166626234343165373463373730316432343835663033 -35313733663930393066353737346432313238666261626264313864333536363035313730323737 -63303631373762633733333738653961363338396563626663333739393464333638633537613431 -38323266666162613365333936646533626637366635613537643861326432623239616237363965 -38316132366636303335356637653431623737643239613863383233396433343936313430323165 -31663734333861306136613134363733613862336162653761363733376132343536376337373033 -66306465666632303230646565636464666364643564396334663830343739373931613334343666 -65343864336134313136653137663764346530346137616431633639323532356636653138633338 -31356266336431653333663266306230626630336532646634313630343866383464613565353830 -38636339323036623162623666633737353166306565623539316336353130393235363133623236 -63393438396530303430356533376533313364393137333533383234313030653461396563656431 -30633930356166343966636330373235326638386132343437363464316162306236626564346137 -35323862326538346464653830376232363963323038303163666435326561303932366564366634 -64636234383437643834383831323032363937326339366335663938663436376461623135323064 -63633532373531633462353137383566306533663031376332666133613332616466643131396535 -61313235353238646661346164373536393239343132666664396436396439383332353366383061 -31306233643966316236326137623763363739326562346137336139393537656639393762306163 -35643335336463616238326530643561303834653761656134613034633237363232633937316539 -36383130383565646237396263383964376432346639363336393964316633666138626232393763 -31356436356337386136353239396331656261323564313163356630373230363537306334366462 -32613638643965333962336163326633383063356236633566663635353866323262306666326364 -62343435336437326337303031653665353532653930326535646161636133353364353761623736 -66633366353266333561323836383663366238303162373362323530333938613636396439396362 -35616232393830373632623335356233376239633633316165393537636261626664666238376661 -32393430363933366130356339306333623439633862346630663334353839626664636365346633 -64306239353066323264303266636432353235323635393063323130343838313533333661336236 -34383030333837363834653239626661373030386563623064386437306631353237333032353630 -64323333306438633136376665316130316633333962326636653735663730333539356335663539 -38383463353038363265366138633037373433623835663666363236626133653563663236663236 -38333236336134623764626231656166366165623332383933613932353262323262313031663862 -65323563313634373764353565393465383361653032373161393962366631353039366332333536 -30623932623966616463306665616133353032333235383965336464346532643830633965636431 -61663132343336393035386563333033653735313866633838613062653931343237663139626265 -37373434616532373762313234646135626435643837626563323566303863343263626632356232 -38313733626666363064653461323466306161366434386439376530343231353065393066633739 -33366432613561326665303730396362346236303439626531633262643465653263396466623361 -39306639623465383632313335656336303262396334636663656532373439653033383034363563 -63623339373739346364336530616330643766303865326535306534636366643134633232666338 -62616466643633333964646631613866303462386366346533613061313064383535613361373934 -37656634353136393665663735306335626265326164316636396566303065656432313164663535 -31626132376566633164343738653530666438613034653433623031303232623034613465653835 -32613234616537333233333933653431306536646337353934653863313037316239333532313863 -62373264373839623333393563663364646437373061663439323030303866323630316431303636 -31346138666162626638613436363163356531326239656238343730643938313938323665623938 -38323932353839363665343463343031336438616132303232323531376561653265356238343538 -30646434616538323632356636306235383736663132643339333939333531633263643633363362 -61343439383035336162633763333762383535356564336130636132346532646563346266356131 -61336138646337653739616637353037343432633265393162626536646364383931663034313762 -66373834393363343963633361643634386465393432356137343235343130623166653632636232 -64366438663135633861633132383163643163316565613465366361356230643263633530353837 -62663766353038376366663837313964373164666332383037633262313635343137653536636333 -61626261336431363735313238396561666433643037346133363336393662636330666637333762 -39656466353038666339323833613661613562363931623964376134346337313136613633336431 -35363536303830366134346433656433393934623465636163643166316461393163356563663832 -61393964646462653864663730613066303861653938356538393761626566336635326364363237 -62383965616364386233663930653732333730336466306636316337303766626337666434633766 -39663764366438646636383263653330636432646230363032336666333930386332396565363830 -35353861663036343534333834366164333862336331333963356261636531356262616230663034 -38353032356535313139653236343333356139663737616331386463316565396264346336396639 -63316561376465303765353064363666373233346461633763616566313765323031356565316531 -64626636636135313330396337623134663837633530333462656438366565383032643538383239 -65363161333866353365383834366666373163373866316639353761333164363930643532346165 -61393466636564323833353262636565633764393434363336393932323962373132613138613630 -62663333393066633561303331306534376435343962333231653635383530306238326466333061 -32653765386265336361323432356465353334303731633565316265623631366232346531326166 -31633835636130333030626136306133656331343735336464356261393332373530396339303630 -31633966353736633730303161353562373536323230343734663430653565373431373462616237 -39376335633035333937343130373432303737366334653564386632316434333762323639656139 -66653062623430623632316464353964653431613538343630306261653134313130656466616335 -65643231356232356635346530326564616166656237663264386362356432636631316461396166 -31303833653165333535633633363936333438326537383762613131303338373634336666643535 -33313230336137383866613632313534303038643838393136353562643234303964376330366231 -66346365333261373030386639613536323533373930666439363263323464333931373635303539 -36336637356633333762363038333935326633373032663862336438646537633539336638363264 -38333263353537383137356263333235396163643836613334366333306666656132626661346433 -66613361666134313036383661366139376561363337356265656239393037633461656538666638 -65383165373438353334663564313637633235613663653239346165333864343363626365653131 -33396438343838333638333066636231326437333336623234663863613331623935353330346365 -64386134646537383365613136373630323936633939386331313066353333373939353732633833 -39643339316561346565333033303361316530343466363234386663663263373531666565383332 -61363134303930623362626466336434306539363266393238653937343132353134323265663931 -35396232356461313261383466373737613233316165346433646162363265323934616263353839 -37376635626239333062363238326365313332356361643539623164366463386264336665303639 -35623931613237353564633432343932393535303566346163396131353233323131356435653366 -35313933313538323132333738633730323565653666376362313839613136353937626666313232 -36663963643163643062363161643832643031383434393938653630646138346538343939653434 -39623536653465623263386465663432353338356335386436323564636138346161373339323839 -63396233366466333135323531626563306336363536323434386662626337363635663037386535 -61643665363733366565646232393930643564333232636233653430323838356136343537393631 -30633436366239633630633139643361653565646333353133323435326663333838663733396436 -31333536326537636135626538373362633337643432393764353737613963323437383530616463 -66623737303461616338666236663434383466666366396264623436353739616233623131363766 -61643461643538666531613366353863616234336233336265666534383836636565666437376330 -34333933636634636463623332636334323566343238663234336563643463363934623335363762 -34613632326234323566316433313563363333633633636362323161313838626635386535626338 -38303265663132616639613866363139396361326232613436303963376535353131396664623732 -38373334623931633335613237343561376365333937643964366536623963636330626235333331 -62626662336332333765626538336238353862663733376331323563613662333035623864363466 -66613762333632396433633830396261376166633730386430646439383131326235653930633339 -39313762343839343463316432313461383239363062656362643864636337626236393166663735 -30653736666438376665646135373739393438336166343366316664393966353536303334333732 -34636163386165346465376134393936313563373965326263626438373637346662303831323864 -63343037373663353035613136323133343630326631613732626433343365396365633763363136 -38306633393362366434366165383935653732333133323336303962303533363262393533303163 -62643562316566346239633361393535373334313935316235306431623133636132613733626538 -63646432343731633638653166363736383362646563666662353430316561303533393833383435 -33616330356266666432353561623137343666653764653535643261626363653135333964623065 -32326334323761626563356661666465616332633736376230373362646634636231353631356165 -38323233633966316535333863323237306431646465653237633761393064646237326564363133 -62343335366535343864633233323935616136376236313136343331663264333530323832636636 -66313931653936396137613030376161366562306631653630643962393535613033613962343565 -36303361376431363561613362306130313530333231383338626130353539343831326564373966 -33666561636232646638376566386266303730323165313562323434396336363433326463366365 -39643763383366666663646536323432613462633465353530363131316362323533306134653364 -37623835383135396233353163663830356239346633333433353636323133313036643335353064 -32303937626239666434623666343136376263643861633030316633323333346532636435643133 -63303461633038643830363538303939633332623934343163353738643936333732363635343866 -64326263623438336632353432663663613230343335633762303033626538326236316132666436 -37626162653636643030343432396537326639306635363337333164623166333430343531343335 -64643131313266346562656138643632303430343537373738353766353437303339646665393766 -35396535393164383635363239643436316333323937356334356234313430643065386463653363 -61666431656563356138323163653634383738363934663763306363663464323532333333356165 -35663237323636333532636561353763323435366636653165346234326636303038656137373037 -39313938633065326337343966346662383438666664303466626263663630373733326362353235 -62616536363561366332396565313366396331396261663931386663633832643732386334386132 -35393061383134353835373061393061386535633964393032663533303031326163393534663335 -31653135323936623537666638393831636530653534653366646231383533313832326132356639 -62626664666230356533366633353966363330356338356139353161396466633835316537663333 -66633538383663346433653066653739343337393136356533333536393662356535356537326234 -38323537386662393130306635313263303666356665393434363335623136333133623037323439 -65333330663436623934333066313361333664306139613632343033663934346161323162366139 -38633836653231646537633630633062653932303263366466666536373339316430663465326436 -34303836383161366264343363383232643033653231646632363634373335333134333235373036 -65636536383036373563656166333732363166376264323666663431303765643963613031363839 -38396232333164313830653065316430313239336462656138383266313734666463326364393163 -36663236336630306531343636343764653963636237633261336437313538303334663363653761 -39346430613161636233353539343634373536623236383036333163626231666164383137393866 -33343933343437616235393066643936656364323336383563623065333864653364613363663437 -33343731396564386561653963666563646636653937326134333930666539616131633138303665 -63623832373864363331306332613866626133636566613634343534336166633466303763613437 -66656432666661396366633730346532323632376439393732653266663435666136336535313238 -39373466633537643461326634326435333838393362613038383763666139336330323239613235 -36313033306234653036613265373536386230326364373262636134346131346661343839306563 -66326336393838623661666136326661306466326633363537373132663433633432396638313864 -31323532363637313233366166366639363936616533623263393633623062646638383839643639 -65663234373231333230343632313434333039333434383336346137623831313265626235616436 -65353863663034396165343930376135626461333031333339303732636436363862616632636332 -31373130323734653264666361393563643738643733386535316135656335353636396333616431 -39336264333265333266633864623162666464613930313266323333346335366430393366646463 -30373837633963316264363464626139303532656161373334643636656238333234366534646436 -30653364333062316463386234313336346433323838346336353333613466323234376236613565 -36313035643530373163383366623961306262363865343436323061383863373763613665383738 -35363831353330343237386334316565663834356662666138333430383034323436363134643530 -35323462313330666438366461336465303562316632393037633464613835663731373236396364 -31366332636463376466386461363963346535396161333430323937666365393132356535313662 -39666331393966363937326537656634626536306561663163613664663535353036353339656535 -37623061653561376239323831313131633130646530666437633864653463623866623964353863 -38313865646531636534363561343761396165663665666161393962653438353563653663666332 -37386634653563386538386663353632386166396462333764303961346235626332316332346230 -30386234373662666365343361666162373832653733343462666131616563623132396134353137 -61633039306366666532346634396331653835643135623439653532616238633134386361623039 -30633839626136343664363534666165313836326565383234306561643666303261303363666366 -61643539393630653065623834653738393136393465393163373138316536303335623232663934 -32616265653365306466343531323231383830613339353139356134323237383832643465633332 -34363530316137626231333836313166376365393465306334643139323133363931393361353363 -36623866363637663964663532626630613462373362616466376363636466666137396163353438 -66363135323261376332373939653163346135323463396339666530396664386638356566613565 -34666234646265666465643262373639356634633164323831343432313639333765333465623739 -64333532623366313463653033643130613166303832393362666365666430653366326133303662 -65633833313836623262353562393262333539626564663738626534313763663137313066373432 -30333631303864373561306436633061663030346638663431636638636136666334353035343732 -61626238346637663138626133363465306138396533333738343337343937346635386461343930 -65393465666436663966313530633236313266666561663932306333343133396335623831643439 -63393764303862666539626532343033343861343539656637333435623038626564336330333564 -64303936616236366161666332333165636538663537653863356330643238643136643564663837 -36323937393936633863643761346131653138356537373936363936323663646366333634623165 -38353933616632303533376338366230373065373631636137386336336562613663346464656237 -32393439636535323338643664336334616530656333356237383462663463346334356537386263 -37643566366431326130383138373463343536323765376631326338343962313761393765323461 -31663534336137623833326233666530363034333331636466633664616138623236376566366463 -33666465636436653636303531373730626163616563356565643733313266633330333932626561 -34646163383130353933613036326232643034633933396238386366383037323763376336386565 -62643966393165653134353032623431376631613334613630323539393230323430373632393536 -36643262633962323039633661323937663861633164636363366163356430643736653234313230 -39336430633839363863636362643138316437663139313732303330383763313532623963323965 -61343536376235353533303062336139396462653735333131373262363233353465313534666434 -64616637353963626637633238613436613537356430343733643962383238363134326439303233 -31386630363766323866376166373366626262643362353732343366636633616336313138663663 -39333630613835376334616266623236623635383537623834323833386334393365346239663730 -64383635383563383732333138613065383866383866366464303862616436643164393764386464 -63646566313065646233646530366264653035353261336161333037626265306365346463333631 -64356339623139333864376163343932656235626431633333333033636636666265666234393462 -38356336366433303535376663333539383335373934303763613461383666383663653464313338 -62393038343762366266646136626530316661343039326439663461323039653062663034663438 -31643230626663383634616565366531346366633763653139653562353938656530613333333339 -32643438633065323265303635393935323936316661393630666334386139373233663165346436 -39313431383130346266616166386438363661633332623931393362343264386266336661313133 -63633439363631313239346462636332646565366364636162666437643730383261653738663231 -66643965353466666466393036323537663636653230656138643036356562366433353062656663 -63643337393066363833376138343466383032306338636432643032656239613032363139366438 -66653561313232343831623934326337383733333831386537623365613661613337363038613139 -62643330303136613363333239346331343233306236366132613035656531346136643366306365 -30653564613536383935306135313634643638313466376262393839666334353638343061623761 -64653532346363383737666639373831383337356633393435376536346538613435336164643530 -33663431366135306666353533663439623432306333643834323238343332303361386238343636 -30626430333563346538633235393735383234613631326537613964376464366366373266336662 -32623764656432323831303864383530353433643834393464383064343533373666643833353762 -36363635366638363964643132313633363534336663373561313739666638353434666231356264 -39316638633664313333663335373761663037373665323937313535386535363461356164346531 -37323330623263313863353634323536393430303938363337303433656364323463643434643833 -39313963373166616235666430663362613539376161663765396161663565356230303736343239 -64626366336536323561656661636537663033613833363232373639383662623236636461343365 -65346230653161383038326635313566303066363138643363363934353363393765393537393137 -31306162623964663730366362326638633566306231316435326530356330333833656564343736 -37303836366335303030653135663565356533653337653738303631646437373361333839306466 -32666534363039376131373266626263613836333730633765366535653936666638333133333330 -37383638613634626431633933636432316261623734636165616662303966333165353432383332 -37366565623262663731356639383232333963633163396332656530373063373864343334373132 -35376436356630366530393239396437653566653966636131643034306531643836333736373432 -39326232303531633931356461636331356534383537376363366565363063393032636136633433 -65383864323934333332663939316333613739336532343266643962633963363862306435373236 -62623737366532376633323461643563343234386337623230386562393239623437393666333733 -31326266613036613936623661333764333239613734613865343031363038653063656466306631 -31323663646132626565626662633265643461323836396431353034393765353463363937653634 -35353731383932633831356439346663376335383862363737636537333961326662643739613665 -30623765303965643864653463343766663738336230303265636336323134663665373130326666 -30626366613631646432666530303964356565313861353962336361376262393737396631646161 -66346661316634373734396636633161343233373535326232373566356633643431333435613437 -35346266373539656233666338663438646430316530333732613235643330363438333562353338 -39333337326361366536393934646666333264373733356162393834313331643737353637636466 -37373161303737616337653331386661656162613830653563303266386261353638613436373465 -65633961666237313564323831623435313434613966623337623961373938306633336331666363 -32633637626461643566386462663134653864616530336564373634643330373063346466336563 -31663932356430623032396162313062303335646463636130396139396134366636653133383431 -64316537383465303664343065333161386134643932636463623962313063633835376132316135 -37363638396639396330623933656561613734356233616531626339386131663539393034636366 -38663164373561396436366439653234353637623061613934353132353335636365616663396135 -36323535663164666435636666663464643535376464386165363763393535323637663163343064 -30613761653562336261653839666261343434626361386535653435393636313263613433373133 -32373234633739626336373131373662666230356233653661643632343437393965323166366264 -61643561643663316133626433323165376233623665343262386336373737396232633431306537 -65376262363262363065346130343931623731623132626365383531313537346365666339363434 -39353834333263326464306433316263643465613964643538353433376639323661383433356137 -30363037656432363632643661353866636231633935346365313131613930303133633736626262 -30623964343938656266386530343137633866323339376432356264613730343564316134646464 -39396133323161623564313033393831346635636139376161623065626466366637323532636132 -37316236333366333765646166636565386264613161666130633663383065323462383530356664 -39373466633231306636623661626461653032386663356665656437616632366630363035626337 -35343966373537666531323234616135663864633266383738373434356138653736373931356561 -31646665323630356430633633633161666330366662386562326361316234616131376565633134 -65303836373930353266616432653762636435633865623439303231646338653132333732336430 -30646539653336303330336335323964636434336231366330313836343635633233343534666137 -61643538616365343064666337333061343930386332373432306231303662633162636434613638 -63353630666237633330343032386335636266626137663265643638613334663931653939333430 -61363865633465633263303939333033633062346536616331646539376263316462326464663838 -37396664373137326131643564363839383132373934363765653033643939636239353537656530 -31653166633138303334656638343730356361316563653839363461613634393262366132363833 -65656233326163316664376630643661306537666437653539633132666231353837666538313332 -62623736353630336433653964616438346232343733346138346561636566356235353738393935 -31303362376464333331386633386437643834376434653436646634663635393830636136376533 -61356238316663333539316332656464633035636430623533303862323632323339306434323362 -62323831653431646664643832373861623562656635623966366362326366393738393563343030 -37653766663563366637633163303962663162336166316131383063386261633064313266316164 -38313065633338663830616464663438393535626135306537376561613733363733376362666161 -63626139626665333961323261323162373235323039306433326461366163336632653161316236 -35333737363234613536393931633931373333353165656631336236376634656530343335333263 -31613631333865633533363932643336336434313361306136343762646361346363313165396139 -62386237643630386661336366633466336563303334383363623637343063353365613635313733 -64663035653661663865383837643464633438653638623562383133633839386237326238363663 -35663461393230336438613337626165386165666630306365616161316635633339306162653765 -35376132653735353161313634393739316234626135343634636166663337316531373732336230 -37383966613637373734356562373534303264323535383134313133396332643830633566303538 -38396561633233303832393634343734616265376538653235326132343433306263373032646462 -61633437396166383734613739336530663234306163343137363366643232633966616132326231 -65353630396535333238356263616163663963633033396435666264303434646636373438646535 -61643336616565303133323935616438613164306535646438313636396664656564646239623065 -34616334343139363137656231633934343162323834383830373933393462303637333831333836 -31386236333635336138663033333137326263623662633766636339306431613263316432346564 -64393733363731636633616465353135623333653461376164323036616261666131396637336337 -39623336623337623531396162386434333562623838613264303931393636386164323138396363 -38623133363965383366303262663037323763373235376261383039646337663031316334346137 -37626639323438393330386631363732633331626465396362613334363961643431326363376330 -38623730303765316266346464633962393539613432323931303765613331623835646133343630 -35343938303532326630303635356239316532383963646631616464376639623933363630343264 -34373534316537613262656532663332343939373436396239383935343733623061333531653763 -33306231346536656362623765303361376464393738633436666533653962663136343137346138 -37326431613736343532343637346332626636333664393464666134613438653164353066653933 -38383235316632663566316538353634383135626433366436353332653338613863666265636239 -36353838336662323764636361656533323339393034616662613364323034363365636235666264 -61303232666534386662613739386163326331633230653465663161346432636134393733643462 -31386665303731366363393232386364386430336536366261346539643437353936316165376438 -39363865303830383463333362316563656266393737666139646235303230633432633939616235 -66333836623461636430393465376439663038366138363239323236383864663835616431613132 -33663062343865626635356331353037373565636161643039386666366463316638393364356266 -31323433303363663366396639373938643335623765343938613931356261623134353961646237 -64643964336234383461366639646233333964643733643164663962316433633539616263643264 -39326235633835666530373763323336323131633562363764373462343731356233316165653963 -38323065626561366166326634303931373036393435613731356533363431323935633337663932 -64383930316530353861333863623831636535393861663133323432386134386438633737306262 -63613939363538646435393837633833316265386536396666313130653830383535613834373830 -61386463623561623737666639373963383233353061663065343865306562656633653339306364 -62376639633033626431336538346564626630336638633934376163623037653736636237393866 -66646433363766656339313137356537393237363266666636383238663536376130313630316432 -36356338316563666639633763343438393434303437373165623331396135613036373964666632 -62383666643134636633376661666339663832656461363832356439366339363338313036343130 -63353438346265376638353635646439666361373161393439323832363461663863666533666434 -63643131323831313837333931633762343330376164653264363333663136363539336637626265 -31353631643566643037623664333737363836396436643836666364383939303737363264346239 -38396338383031633563303862376632363531346539613334366562323137623833326561383739 -38616166316663646133666638303538336163616634306531633533613835333838313931656564 -33316333376461626561393638373031343535656530633936346230633962366164383061653738 -38643862306564323266656433376139646539306632366234363336383132633133663237663331 -37386135353566306564326630323935333861313434316264323866386434303961313763386230 -65333331393761373862323835613663313135303936353534383363626533346666373133306533 -61383237653935613737396332343737613934613636336530613864336133376138393932653630 -65656233376662613666643731653062316238343532306337363162383030326364336466616233 -35373231323535363839633363623630376639343966646333626235613362373131323263396631 -61366638613132303265376266343433373663386231646536316463656530343562656437653234 -64346639356563366633353134363239633736343834636337343534363537336238343965333233 -33383833383364303461343434343638393162303566313962333933303038646235613631363863 -39613937623834393834363239316435643538613465393334383132386261336164626164656462 -34303231333431666530383766323564386336313035386134363838613733383530393561363136 -30386466646238366337663731316537623965316463393366303166643862353064383236323837 -65643262393463636332396532373037633039366632653838623063353363616239393530316636 -34303163303965303738613065616266356336643334366531303765643764633236396239626334 -37313539653466623334333933623664646535633166653666363538663765373866653633326335 -38356665383438303563363062336238313461353339326361613161323731663239653166306238 -65323533363837633536653261303138646435376162653031306238613236363333326231303035 -34653964386164653465323039623331653339663563303938663161623437313064663263316238 -36613738383631643839386236613835643834313531323639666436666437666435633833623330 -30666130323062633138363464373532643531613232656164343963613966663332303938646134 -35616337373637323464386662663765643062336665623432346336343330363233396433353936 -30316437313038323735396365663930616232313235653231316666396665316134376634353037 -39343133306638373662323732646265366563303163616631336138653232643631356630373934 -37636161646161316431383262386366313536333738323638666530643665383839383763646535 -34396364333565646636343033303135323231336166333936363738653434396237666563363532 -61376137623462366162323661323535656666616461613334396233636163393637393037383366 -37333536366437303262313561306337323431366161653461303535636264633238356137326161 -63353438313038336262656261396536636335363830623236373632376431636137386437383365 -38633832323935376336373365316436613861376532343239353563383534396132613430633333 -31313362633732346330646261303131386632376635336435313737333730356637613331333739 -62346535346638306632646562323237396630303434653063373332386135313233643862346530 -33633837346137653033353561383063396364366263373662353434373037333237373834316137 -31303637393637666162313266656333383163633638356163653366623061376561333930386330 -30616630623062633336616462333533323166323631393764386562626633366561333238373964 -33663966643939626164643664343237373132313066326434376332643734663631353335373437 -39643738323866333230353763373531623635633165316665336163613965636266616364353865 -37353137323239393430363432316666336238383865346438353132653461623035333338646232 -31643138353037613366613962666338346566333866393862306534336630623837616234306633 -34666337663562356262626238343762313738666634386236373966303332663436663735383930 -63366663373134373062363532373835613166343563613833303264633032656336363731666335 -31653834356661336364613763626330373738386462383432316136363563623466346638356234 -66353964316532616136356532313431623436356665313939333064666132646630663034336163 -32653730383763653934646531386435353934313830346162663965613432386661643534303865 -65396133346161336165343563303439313061383863363632326136376461336531303764323362 -32366361323666656666343261656138643139353738663338623538396461633434653833373532 -37663338326461303834636466613336343962613638633365343235616131633532373532323230 -39613165323532356465616330316534666137363463353535383334636536643030633136366334 -39613939643833626565313232373438346261343861303965343365326633663565343836636239 -34663338633132396634613038306366323262373530363939343961323131623161343338373166 -30343230373938383939323564643366326335356462616231343663656465373064356333316134 -61326362653263396430336439633961396637303865346563386462353434356363613236383336 -36383639373735633735623366393165366263656438663337393339383435656136303534356539 -36313534633064656235326432633565653831346333346331383836373931333664616665643632 -36353436383764653037353033373261666336653364653762396635323839626236656465626238 -62306138396434316163383234376631373936306238336436646134326338656565646133633866 -31623832633132383965316630376637633764623232663631623065313637376462303066363535 -63306335626430383962393230333333633664613633323038386432313738666636336461623461 -36383633636330633162656365353039636434666566636233353039316266363636353336636336 -32343238636136613561303861646531653939356337303338376564313739626233643963363564 -34633036323064336637633266363031393666373231643936356561393132646435376330633861 -31343034363831663134376639646231383439346531643761623635666564303733643161626438 -35643964343436626632373561373137656631363836336665343361316563316362323630373133 -61346430613330323365303734613735366561313839613134376532626339343065373538613433 -66633732613865363562343165373935306630656666626438363933393237356438386534623763 -63613836623730376565373861643631383436383064316361316665306336663263663636303130 -63306563613131376234333838656463336163643632303166386661363065616466393066613035 -34663533376133643237643331356630616665396236316436343661346337613638313334363962 -37303166346130316138396137643230396666663437316161366461303364323433333132643931 -39356435383661633661636136636637353432386665613966623130343164616431363636366630 -33623339333637393930643530306638663065333037663664373135333930303462333836643332 -34333737353136633133636231333130346130336133383364373231343830303832393535346138 -63663230363034313330366564653330333863626237363266333938393261373439336562343835 -31393862333565386661353563303132326332613063386433393963343036616235623761373332 -38616531663634613038363436353961353065613039353734373664323234373362323139623030 -36623735336661373037386261646332376664613137633362633330383035323530633165633637 -31383663316165653163323865616330333337363633666430613435643334373366383464636437 -64646233386239386464316430663063363738383962383736356130333933613139633063316139 -36376332616237393233313064393661316362323931343061613038373137653635613433663836 -62333764653937626532393330336665666139316566346330636136376237666162386233313465 -32363534306630623063346264643430373032656464376130343332353161366630623863343138 -39653230666238373136656464623139373634636434326535373339323839663138376361623061 -38613834656238646537346439323666396264343661623438636238663464666233333837343838 -65663761363562316133616664366139356435383138313566396135623933333265633165353138 -65396436323665316534643838666231306439353365666439356534613863313736636539666466 -61336462653365623933343232343139306636356432393631623237643139626436373866636233 -30303836383132313962396164356437643362646565616461346432346563383333323032616430 -62636331643231613065636661303339333930373231386635643066386237633834646565376664 -36393430613534346632313862646638323931376438376461363338633065623639376434313037 -30336230393335346666666536386539616132326233633665626466323736386638333238356261 -38663935353762353965353338663538343664656231653739386331643736613564633061356638 -64376337613965633239373633376535373763363037303131326263313630396264366566636466 -32616165326435303235633734336536303062313533616231373037626535326366336232386538 -33643364386565366134 +39306338336433326261653336326638353132626532393335326631633936616235666462613536 +3663663237656630363262636133346133363335353366300a396337613165313864343639396239 +31323565643835353462653534386662366330656366376131393162393933336332653531653165 +3830643864663961370a633661393663306464303961323764383938666165643237396364643439 +61323031316232303235656531626438396162386163653866356535353836653238613337313565 +38356563383561626437373965343634356563316639303536313736313263353166643662323831 +66343530663631633833313333316239373764373839626231306436313764353139313733613930 +65303339393564653163613139366366313762373561393938376662633865653533623833663264 +37663234303537373136346461656632666333616566663330623536336433333162626138303662 +30643531306163366462303732356231313230316166313135373630386163396433616634646261 +64653930343366656531316663363830653965653039643038343936353962623439653138313632 +64626435363565393930613831316331663166626231356432336339646163373466326530626266 +37653933366133626264343865313139633831316139663731623863383130613032313738306464 +63346632313838363733376261626430613161306364383762376136343536333666396565643937 +30623331636166326631616639316438646131613766366263393031626562333036323733383737 +37383131653663333232613263333836636633346232616430343833316366363236323833353339 +66653365313230643936336166343231636265356638363832396266616162313135636534393137 +65336166633235653337303938666432353538643237306464373138343066636566613164386462 +65613830346162383030353761366639623832666539643665363933343036626236343437313733 +63333336333037656138616532396363313666663239373264373065306432363731303434323635 +63356637316239666337616463633537366361636430373061346338666433633938666530336664 +61323034623966323663383530313838386431323433636431666530396337666130303636666636 +35666163366563616563303630613837393930366166653066316361336566303334306165303866 +38353762353136386564656162353234353733383866636138363432383631373936383935346630 +61666638366136623038663665383534316565663536333133363535346461346139663132633065 +65323165643565353138616261393566313466353933653033313232656265356262313834643033 +64376233636565366334613466636332333064336464656466376137343665353431313161636237 +65326435633235366465346564636439316264353637663766323735343830623830626631373365 +35633136626238633731386266373238323165363737633235623763366430666434653134366236 +62626162306465313339396461373231626233306236323137633362326538626331666532306637 +65383533316562663436353436643763626664313039386363626566613061373365623937643264 +61366634333136616233343462313331666635646262306433373638616137343534303366313761 +39393030616434373434393064336663633166393366353031326230346265643336626466306662 +33323834666663323763363864613264303061393231663262346439633134356363633731353332 +61623362353738323130393564626266396365356638343164633933653331323938313634343037 +66346334393837343738653865616134633062396637393733633537396665616638316264393431 +65356337376234633634366365326530623536303034326537306634663730383039373338313835 +62373065353133396632663536366664353331303833313463383133663563393362303265333639 +62333935356361303864623331613239646439356665633261646664393136373133363665346637 +62373163663465393535636530363535386465313464326266373636356138643238353536373830 +39356531343131646338313833376138343537666137616132663136366233313064653564393161 +31626666653437366231656130333066323363623732653766353365363838343930613138326663 +66393864646339373065626235356132306135353137323334626461313935666138623431396132 +31353235343938636261666439393465616165306633343838366465353832306238666539366230 +36386364393933616231383366623234653738373361383031623733326432373531376662316164 +34623639393464316332376466393537396332626561663338656538383839393961373436383331 +31623435383564363637663135376334363032623630616533393835393632343533326339633137 +36343065326239393661353637653436663531653538343436383734343634626130323965626666 +36623334393265353134383666636463376131633231353438313563346462646635333061353634 +39343365363936376362626331663966653137356537383735386332663138343732656663616364 +37663639383934336163336636383431333439303735333239366630336162326565313330653439 +62613963343561343931313133636362333863616638643135376663613963333230313437396531 +62643432653230303834353236666236356262643637623932343964316666386332363635313265 +62623634623161623030313262386266393532333236666565313131323036633033373666383466 +36663864616262633663656438396439356163613738366136636365666564383131376361356661 +64656430616664343737613834396239613039613362613835626336336665353535333038383039 +64346330336333616134623665646239363366343038663062666464336134363066363939383365 +39333232636132663266613530316532363861353038306331393638323538383237363965643862 +64626638356532663463373633316130393835366332633565363664383839376365646238663330 +36653061633734643539633236633661613663333333333835356130373239663566373535653035 +32363265616232323262353361643536363231646237313431323664326331663439333631663765 +62643466373264383865643033383533666136346561633230653334353036643438386333366635 +39346133636266613932363634613066373665363963656465396332616638383766633862303965 +32633139323064353534323765333234346461313239386638613137393230383730633838653738 +65373931383661636635393032633566353663653764633130303731363733653130366135383462 +64326333323664316638396231636434663763633130383634663664633635663662313266643831 +34633233653634336136303938386463663338333933356436626132323664386533643265313962 +62373935363739666137333766303661643431333038343638303835376261336335343433313564 +63373933666262343935393565366166633034666665393034376465353438303132306563333863 +62376238333035613261366666363430643137346632663465363135383433636566313862373663 +38633963613039396335636263323865386332323338393135386565626635326431366632376365 +34643964373136653461666435393538326534353732303861623165333666373139643466313262 +35646633666537366264373666383163336138303265646137646336376663663633323833336634 +36366136356138636436333662343766393337623237626665346138633030353763376537656262 +62653263373337313732393065616535623230316664616633363732663131396138383539366135 +61343838306564356339613063396661366133396662323361636265613230346463363033646663 +30306530393932383732626533366432636461326265656334373738643933363762643762376637 +32393034393637363830336466376663643630316330643566373162373831633537306266333163 +62373861666262343135396165626537373064333365326164393732656233373631656130663564 +31386237623236393465343638363964636539653965363030393232386565643065613332346232 +32306465336561653862333131633339343465373565353433383564326232366431376566636661 +39396339386238643334313639663132623739313930363039616264393938323866303333383365 +63643466313236383864323364613132363966666230636534363639386262636664626163653161 +61383233393132633862376566336235323363396531313837633233376330643432393961613131 +65353465616331623862336665343638663639396361393839626363343532643138393066646430 +34343632336537393534393338373832373238663032623166396564383139393233373734373037 +32376335636238636262326265303933353061333639323936346639316437613032386531326131 +37653762326432643130313930383165356437323539316634343330653430626231383165373437 +31643461303834633037633233383434386366326431323630316262363030633535376561643365 +35316164376331636561663161663236343130386165343430333365613364313937393832616530 +39643762333435323065396139356639363935613432653364356562653562626164326233393035 +65383634346365343362616664313538383538643065363735363165656334356261363665303464 +65313334313535626333326665336466306438363635343539376233316661336439323133343731 +36613638343463363566393061663835636436363434386534386536306139663833656263313138 +37323063656135633130373539373764653662663830623637336365343064333232303263373830 +38333261626430633438363633313539313231363966393739663161663366623964366165353164 +38646637616635343237653762383132366565333238373439373464343534303639363538636631 +66653663376435396664343231663938366333613131336434343032396464353532663838343162 +34303131393262373462386161643432646234376337616435373439633864303739333263363630 +65633066303531633434666539373736613630666430313464393766356531613165643066376261 +35373837636131373935333037373732383666656431613735643765636161386234376263373937 +39396534623433376365336336646236333533643330323330356561393331346462373833396338 +62363163303765613136353034623534396531366534333061396662383636663131643735653133 +35376231666530326635653931346134636336353334343436313632346431396562383339653261 +31373132343461363064666333353066653136653864343361613132363337363338376433616431 +38646230376636303231376531393737643436303866326635616530306535663130316565626561 +61363036656235613966326332333136386430323732343465633165353765346661346130386338 +62363932613839343537363462303032616563336230383035633735376233343832613063663165 +31393961306437623037386335643830333134383162363831333734626366353430353233343931 +31396536376161656136363039386362376161643832306263623737373863623664633032356634 +36626431323832356330643931623530366231626239633138363661313231383734666438383166 +38363632356232323362323263373330373031643866373737313732343366326531336134376439 +31346237633439663161636536633361663830646533643862383964313736356161363761356563 +30396130636561656238643032636166616364623539333233646464353364313066636132326563 +36643830313666303431333766303962633138656136323637616335333230346330363231633562 +30303365646235623738383038313062663237323634323364386366353661333661346464666631 +34353132376635306231363730623932666231366335366366343761303532376633353464663961 +30383836396461313264333735643361346631386166613035353661623364383933656165613861 +64316161386662616361376234653634613835653536393138366133336466386363303539356335 +37303438376336653233373631383061393262346234373663616536373235396637306162316433 +66323863626239336130316138323363363033353239336465303563363766386565643830353865 +36646464303635316137376538353435663334613466356663336462613466663466333632636436 +33393537383961323634373235383034303665616137633861666261306631656330616637336532 +66653565663263613762373937633238306164356637316139656538316462663934663663366230 +63373966656431623831383431613866363265333262323462643431313931623631636664626265 +32393263366466653432323064316532303936346466343830323231306633663330303236363865 +38346531613439396336633838653238386437613739373263356238323136386434613932353164 +35353638653931363866653462383738326335396265653637626133313238663538613263393636 +66333265323332323562363665313738656638643565336439643137383837386534633866636565 +33656661363163613133363633656539356363383666383237383433303239326361366231623132 +36666537623062353831336633613762633061343862323964663866316635346533663735346535 +66643866363833633030313961363063633764363065656631363764653736333737303637643238 +33643862306332336335303364306362636235396635633339326131663966316532613032396164 +36366633363463323263373830386266323565643465343564653436666430313362323463326238 +34653663626436633833326164626162383066306562333734336466623431643463363365623535 +34613333623861383365616635666138653733306263363838316634663162613961373266643139 +39656436653563643930646531383636323364353239323038613462363337386637613762336234 +63663037616465666131643431393663373533396136366131326562633061636663373965646437 +62626562666566626638646535363163643239363834313034323362313132383038343464613133 +39383461643933653037346163333666366130383031333935326266343736333232353531393032 +61643263336434383835346666653437626266323961333131383039663336633630333930343266 +65376236653431333832626135303966323362323236306366356266373034336632373262663461 +39323761393930643465663238656437333030333038636363313337323739333238373030303661 +31633766626165323532386464633164313465306336616634353265636661313039663164306338 +34623061303637353831643265336462663532363634343164643463336637613534376465336538 +37313563393933646166633861656431326436363166626132626437373566346639616161643533 +33306339626365666436306330323231356164323336643062353061386637386635333230303665 +31666130343132356565383431643039393934333539343363343366306132386233356432323837 +66373866333361653231383061336130356130323637356264653032303030376634653332316235 +35353839663966343033643361643261313331316139636362383234316565303164303765623835 +38396136613964623839643761306666663634313063323539653635623033616435623934613462 +32613661393164623762396534306230343962353566343933396334366134396563653930666336 +35363562383636303261623032626136356132626662376136383033656666353039626334653736 +37333761366530383466366139653330643961323163616432613033353234666365373531336332 +33316363653830363637386161346438636464616132633535336564363262333461653964656238 +34363865643438646437343134323362653264396132663132346462643132623335643232346535 +61343636643131646538363432613731666430333666643830373862613966323233643635666237 +31356137303239633465333139343765366666613264343639356630353536313833316265333238 +64393762616533663362653434303264393964326132643361643832326439623332646661616261 +36323262376161656239363639623365346230363162383533343263393564313130306534316134 +62646564306639383932653637613839393365343061333031616132393734643762643933323531 +31663730326234393332616439653364346138373134326439633464366139626134356636373463 +62316233626132653861356332376232643137356362343364303562333435356434303963343837 +37353536336534383465303637396363663763363738313630656439653564346131336464326536 +34353934666232343331383365333431363364363361366662353037396233356562623736633032 +34653638373263626563393033313637613065373132383564393136616264373430313533666266 +63363336363534333965353638316165613230663162633962666264643538346162663263386131 +61643263663037633036633731376262346661663930366336626261316230363336303763633666 +33643061633335353032636631333031373433326336386463633838363939643636333539613632 +37333137623538333134363038666331333130646139643635323062623433616534313539343866 +32393239333365366237656566353366303938393638356634653231616161356237326465306139 +31383066396661646362313064363664303636383163623134386432626330346165613939613439 +37646338393637633432333830653532313931373139313432646630663161313837616234613530 +37313033363637363332656264366561663033646337383437313461333965396461313436663564 +62356334636530346437616537336562616562333264343830373132633030663131326534396634 +64653236626161656632333865323438653765623562643235386361646262616237336339323731 +39396666636337356662356433623663623436353433613061383335316533636233313164653762 +66656466346463313233343865363139333736326530653438363731633236623866323362323235 +36346633343639366436396662363461613030633934626166666364336266633835643738393935 +38393632373963363065343733313963313662303463373966623865396566316231363731336135 +38373133613639653365333138623638313465343937363633613939643338366436613336373431 +62626133376662333037626236313736363739346430323461303438316236363533353339623230 +36303330373262643439663635393530393365326239646533373365663161366164376166366336 +62333933643630363561386164366436663434653062396237306161626261373561393733653063 +62333438383237643134393938396632346234393937643436366461313334343236396362363936 +37636235343534336136626134613839363537373262313138653331323633316464376662333666 +63633832326261666366383261336334393263373836373833386262626639646339613865333664 +38666331353730363163623637653765306234363464353431653832353035306535383435653837 +65346562623637323662356465393934636337326565386536663065356165316339663536633638 +31383864346361636363346232666162303538653262663230346333393438303237656633366135 +61666666393539356438666662363363393461333135336538356564623135623262613832363139 +37363337343931303736376337613865343937373237306565326466613236626430313065333262 +37623835373038613733366238326431346465643364613633623338373165326466666265336138 +64333531396363643438353266623534353639303237396262343939386437386134373561666639 +61313563633938363062633363396436323366343065343263656337663264633131353631623336 +64636133373663646466663338343638393737663961386337356138663131653366323861333235 +63323761356664303235306138383831623762363032346331623034373535366235653330636539 +38376362336630623166616130333838626430343134333330306631663130323466343036643866 +31663731303530346338396334336631326138646361613639303933333233333038613462343563 +30343061383161363433333063393230313164383433646466626537343336323335383666346337 +38386562343233666330653632616435316137326563356439366266363431666339383039376664 +38316462623061323737343736633133626438333737643532653831313164313335396562373333 +39626332633731343233396430653566376431363937633535666136633134613965623466316233 +33386665323631306666653537626233656137613365623062353138336231643238613031636361 +35336661306135316561653261383961393865306237663935326232303637613164326235383661 +63353434393239303932316466333730313738383265346132303861333832383536366465623066 +62636636303562333038393632653461343735666666383538323061383962363761383436316165 +62626133333161313963386361643032396364613731323466316236643538363039393965313839 +38623062623461663962356239353235326565646636306166313562643737326633373835346231 +63383366366463343932306631303334363639326263616334643665343465326235366562336436 +34306138616132346533383566313030363766643862303938333731353238383137646331326338 +63633032626664373333343534616539313033313838346631323733396538393435316338373035 +62313863343233613566336539336662613639393439666566323936656364653236636364353062 +62613936623162613539663833653735306133666133356330633439353637613330623630663139 +30646239316266656137643834356662646336626339626466353338383061333531633666356131 +61353936666538623133643939393063626463376232323932643432653738636132626362613635 +32366230623332396361373761336664316131383266353136366664383633393137363836393139 +33653138666461626463663964343237303836313037326633633961613135383464643738633539 +64646330636637366637313638356630316635333030613733663836386638343966303761313962 +32386335323438323461393562653233636238313663646362313165363064386637613064366333 +32363761653665303031316263613539313137643234326162366331623566306463353134333563 +35303832313562326634313733393833303439393661646434346638346332373935613765626436 +66653964323733666630316133346364303337376635376638663365626435643764386466373836 +35633930633930353536306436343461636230326462616265363035333438393066666562373862 +64333830636666356533373139656565643539346535326630333635303539613935636564366138 +30613634646366303637633836646138336638313162383530653632663933653235373838666430 +30353839653561383564386538626634363438346162333331373764346133303563346364656631 +34613362333365326461653930356465306338373638333064366630616437366132323464333935 +65393337636334353865343239313230663065353139313666363032643064366335663363306165 +38303637303638323561376562613034616566653637363964663430633062313664643438346537 +62643930303565646238643664313135623532666637396263656330376363633132353634643565 +34616162633135616134636264346366653066663535613063653961643538353666313761353030 +37346536376534353433656535363632373065656135626161373333316637656333653466336239 +63393338656561396365306532396237323633323230373865346439366637393138646134646562 +32333331326135616433333333393631323330366664373134333065383063373736363236333937 +65323334333232336235366339323730316662303236656436626237663533333438346334343065 +32363961363363366239666538376131653462666334643061356230366563623937376635363339 +63363564623933646363343065396263643634613534353164613134653934643765613933663361 +30663861326131366539323835633937313131396638376332646665623339643266343963306232 +39363566656163633561343765366139616364353763366337306339633437626365333831346133 +36313734653336613266393238343665613461323362393139633865373863313038653335656464 +32363063356634363036636664623031353466376533333038323532303232343534373435383637 +62373231373037663235653436613062383666333539343534356634386332653136343931623463 +37306234393763636161356561366366373266633938626665613633303666356137356138393236 +36353866303462646236316330363664313133313037306633376637656561656566343934373865 +65353839386562306263663037373836343464316366363930316662383462323366626532656131 +34323466393932326536356135653530666262323363333363306232316330313966356232313632 +35356333343462666562613461336565366262396436323338363238303565303063626530346561 +66653161636262303265386133616136656431613064326666346239323337336531353733333630 +32626266323031363139313635316263366531383066653766626662316437376534333664633665 +32633736373434393732396562376332653239663664633230653933633666393539373132373031 +63323866343336346135323835373837653339353239323034316232663839383635623934376630 +62376263386664383934343639363236323666373866353132326232346131353632663261616439 +61616462633163663463316564396362383335653934303365346462303832343962323637663365 +63383637313837343338333731383530343761363430343932613733376561653237653261396335 +61373832316433643137656262313631656563656539363832313162303636386632303332633737 +36306337353231346165653230333232383761646565663334393865663361613436663230316235 +32393765386565346461623662313334373763653630616230396431613564396162623332386565 +63373562323463633633366366343831376437653235313334616133666434393435613362653264 +34643430626239313537386237613965303838636366343832333666306561633465383935313435 +34616334303135396565613964633837653164356535656162663731383034373764343939643662 +63616631346530366130656363633763353533653166633763316334653335633139656139333166 +65366335616537656433373634303338373963396633376539366466396665356465313266353263 +37303934626435373733363336373065663561343139643366613664333032323763636136656234 +39653831313735626365643538306664306562376635313730646363373763633438653566343731 +30613939653363626464323131326163336565653433633431393334666165636337663336373732 +39333932613634353132663730313961326330326466663430656136656462663563323565643630 +34393963626161643539663863623637323736303963663039393761623037306330366334316264 +32373831643835383662326461393032626162616664353432616464303732633261623162613066 +34316166613861616532626166353561653431663438346663353536653236393461383465353962 +62313435653638376461313563346335383664643566343833366138663166333034326563633664 +63333635383661376234393663613964626430393562636462376666323933376162303736623462 +36663962643364616466623964633133616266643261373464343431313161366332343533343165 +33636361663133323866383165343365353030613135366430646536343966336431356664633565 +65313035613661613134643166336237316363396561666461656361393366323534633261633461 +66656231663337653863363063356637373539353337383834353361356538313462363931306334 +66623366613033353466643436393832393039356235366364303434363936326638386161643239 +66643865363465356435623464373838646430623536376330333339623030643734623263613133 +31313763386533333761353737613137636234663030353939636433376363313566303832656432 +37656338373965343832386436333766376630393264623833343163656137653562313731326538 +39666332653533396436333434363864323162613934393466303137366264613766396131333432 +37633362623665386535333232366333666638366164306538626239353536333636316630343064 +64313132313839316263336663313262373234303335306263326439613537333461386232363266 +37363066623561343431616433613439613931383861633032653865383031626466386630653138 +30633736386431376237663338613935383932376132326130646434333536656537663831616233 +36303730663235383534616136343339363739643666346563613537343132663766626131333831 +30386434656232333465643238323838373132636637383163383438373037626264346332663562 +36656336666431363161356130313039376166613163346332393562623766376661613131393036 +62366561666662303563623963393162313434363265633332336434663334393131373337303736 +61383738653030633830663631376264346363356137313465653830633130393337363137633734 +35303535313965633033373335313733376635343463303030343534653832343833323536396361 +30643732303463613035613565653639363066643038336634653265666339313365323436343363 +35373635383865623865376166363331336333303033343466396139636238646239333335343762 +39616334353264383835376334333361646234373463336463643539616633396535656465316332 +62333635316565356536636539626363383331633238333838366266363939626331626337316364 +33326264623738333465613236346333626366313537663633353833663533366431386539663665 +37653763656237356562313662336163323765343839613135306636393139323463393831646138 +34346666393165343930313236633462326235336536363830633733333363376563363836653661 +38636362376530303465383830356666353930346265313833326330356661653764363935393038 +31343433356439373134636131396561303132356166353133383364363365373830363539653132 +62346632633564303366356262396266313931613630613338633262656434373366323130316338 +31303963653535323834646538646339306338616533666333313833643034613639643238626564 +66303633346635343865353739323233643139316262343162396530336566333335626562323362 +39323235646633323430376630313534343137313433643234643561643230613262396361376433 +34356231396562643732643239303531326334613135613866316463366564333366623530616133 +63376433303737613332303335366231303932356462306363363234376230613730313464326361 +36396632613566613632636363323364343063616339663263363263633933366638316637323762 +63323733376165663365303061646631333535653165336166313534663539386632336532356436 +36343935313064363737303461356464633437333632376633313832663130373766623331316131 +37343431313835353239633530623463333333633233663634366665336265633865643337633166 +35383935633337633636343032303835396537366234666335343235333433376265326237323230 +32343832383462333164646564613566313365643035633337313839633339626262383837303231 +36343431356233636262303436643065316239656137376366363563336262343338383930353663 +30303332633231363233356238373531396362303962623561323931346163653132623065656131 +34386333333030363132663735366132306561313566326433393831356364386261626331633538 +61613866396661376435363937316163613835383536366535373537333865303666353637313036 +37386666666435336362356366393039626232323638653536396466663437666430316538366230 +33626365316139646434313238313461626239643134366563383566656264383434363430373632 +38663436346634366233353634373539376238343535373532323831346530613334633366333633 +34346637653565363136633535353730343932353134356536613836346332383534336133303635 +39626632333638393137393664343063646464643661303239643963656666396261313339663033 +61646163663762306666306666396638316434613631393830373934346436613163343561613635 +30653961626635366130356131396535303161666231663530636464333038323030643836643433 +31623730363734626337323361303734323232333336646330343230613735643037346239336634 +63313830353032376334633137346465346338343933613030323262333535333630376162356636 +61303032316336366466373164653036333833323166373333616163343966316632323066653762 +31386465666435386634383162613037343966666330646261613231656130636332393061663933 +63326532313263353865306532333837653362303738646337653235663436353164333633656235 +30643430633139393363633031313462373532346237393337396630626164653233616562356530 +33313637663739363163346635643163333165393531633937306636626631333066656133346137 +38326334383835383866386466353961383862303233653232643263343530336332356335313031 +64663164643061366463346234666663643131303465636439313738353039343366333038376135 +36323863326534306564666464353830653731633836623730333863636336376632613732333064 +62363235663765663534383731613830646532346365343461366163366234376432323961303064 +31363130373431383261663235373963386435633663653737623736363166623136363861323633 +32306431633361386534333535623261366134313466336534653433303639303363306562636132 +34376438303033373963633130353639653332336632646563656336376134656162653662393134 +36613634616532383263373030366464383464313834383536623937393736643730353063633461 +30313138316135336161626265386437326364326664366562613461353234393436643035623639 +31663230646335633037663266633735326264303761313038363238313539383037623232663661 +36393935336232316132656238626335356161646564353733646566616561663030663666393363 +34653564623730356463353766633362353365363964303461656464663237306234313530306334 +35656539666231313232613134326333323139343964643335666234633461363833366639313537 +65656366653433373862393639303663346463356339623835333738636534633535303565613938 +37323337333630333633356238353633663864656637356263323938353239313764343439643530 +39343264326137316233333833633836626537636362623461623130323432386366336562393664 +63393964653933306232623932653630643033363339333538333733356161333335366335633734 +39646263316563386631636562333639643238316336333261313131366538363831633532313433 +63356263316366623365663364623066633132616137613233383134396161323033653031656136 +37373130333564346366653863663763646530636538353864313561303532336363343832373364 +63636566656362626565343365343231376331326561653334303466643561333134313438663830 +65306239366663326534306330343134626133313464383662346666636464393637636439643333 +34613937376137333065623563323964623766316134323530313565646336623133303837326139 +38323632616633303638383036356430623562383236306361386433643062353530376566613539 +31326465613330666330373536393733633238633864663836306266386364633433666136613037 +32323534303236313966633065623030356630383363623263633835383066363334316330326433 +32306266643463303837626232666235633337313964643663386464616166663464383131653534 +30613136643139636337303161336130343566653237616261613061353436646237326238373266 +62316365356535376332653262346138323433636236353439643261646163346164626132313337 +63643062363832393831656134366631393131313362643236643461316235653234623664353036 +32636564306334643232366661663533386238616431653863333437623237663761656133623931 +39343232383161646134623834303065336361626338633238353733323363366164336631656232 +64646365663138333030623566353930386334656532636630363230336663313266396464313031 +65633532663339646231363930396633613062633334626534633765623639346134653333353165 +36303736326438623762353939336334333935356561313231656664363937363237346431383833 +33363365666563383464356538313435313833623338306462316635633561656562313131643038 +62356462633561323036386239613136323634323633326233366662346334623730323030613134 +63396138346638626562313263316263616665646432376135333534656535313836643064623333 +64316230616634666263393037346266303139653133386237363932376563663530383235343665 +37613834323936313030306566366638393063656463376266316635393837623263306135646162 +31376434623161643530336139306431646636646331306161333034663238626337333438313930 +39303835656264626537633935353833616163386439333830636362353165613261643365336239 +34343434623131353463376331326330376235633663326462313835376334376135376664306564 +33373234656561636331323039643234613234396434306463653233636366353562343665363630 +36366461386334613463396537623064346639323964633035383836613531363966343262386339 +65653939646332346635663962313933383963376134646237336161373730343432643938376438 +35326365326664306335643963643837383766653834393630633161353831366432333438383165 +39323532373839336566363037313536373262623135323036376131356135393534646566303765 +38313534623162313233326463326637326335333930323731316633363963623561343462643533 +35356665383665646433333938373166356531373437646564623935393836613366303336636666 +36393631313963373135363365313062633738393038343539636665363337343463656263306532 +65613561336539313537643632393532303866393838306232363838333566343161656430383133 +61366537326264643631336535393830633737626637643239353830393831343534343664313732 +30613432366161646432373738393338373031326131313730626363366662303138373463393061 +31336264346534343465636365326337633030383531616364306261653766316232653039376336 +34656465656336386139363637373666366137343132636437376631303330373562333533653539 +30336165303163383839646265333466373135393635343939366662663263633765376363626366 +38616164343937383134663466313865393036396233316230393238663865383763626265623430 +31666236623535663265616530386462646331666161656331616337323966313265636262663336 +33396637363866366163336333656664636262376335363437663161623036306435346166616437 +38376162316564643165653965666338313234366364353239383065343230653632643932333832 +61326530636363373036336235363365633066636537346433303638303938306530353535393265 +63626138636632313636376661313439366562643764653831333832663931316361326634613536 +36353163623230343066636265303639353238326331393264656338633137316433633364653662 +64613536313366613138376665363232643732373834303731383964396535626262623865336430 +31313133333032393538666537623662653065366365343266363338303163663364353264623666 +30643733323934626437636565303134363732383137373837663965316536653231643632633263 +61623064343761383661623863343735323863383332383962366462343434343531343966333533 +36633165313565646538326234316634666164363863363139623237396430666232323736353061 +33656465643361343032656438653663313736383238356565636431333739343938326563393935 +31333665376563663232316363616637653866613465613663373761316230356431323239313736 +38353233336331343731626464373863306264623735383736393539316633653463666462636435 +66636263633763643337636135613030366134353765346263306565366361623639363831386636 +66633666323833346463333433313161363436326532613739656133643532333761616464343537 +37613166646238643431383539323438326163626265346431663437336530313633616166373965 +65393037393464373465393638313764326562306164663234386137383833613562633961663566 +37346362366261323962326161653365643331376233356133393062383932663164616533353431 +63636665616638316337386564386539623466353637623539636339383033353435343637666363 +66613566333566323263396463326333303038616565663137663132343039393761653638636233 +30393931316561346633623731383132306238383634323738353266653164373265616165643238 +62623338613338366135376536306632653731363636373062643237663433653032306234346465 +30636666666232343235323636326533656232313738316231623538366230333466393537613637 +39333333393565376135663338326262316135653435316566356365313161316466303431366339 +31623338383130613333386265646339313863363038663662366236313531333139643336316463 +32633564333262383930383763643832623539643662383035633730393636623230626231623833 +37383434393037313535626561666134303165313365343930623165336237396263346633393039 +61393237653366666337613035323332323863646239393662646230666139396339623836666562 +39643439373764323136353161303733346666626561313234653938623034346262353464373833 +63613434303037353562313335393937313237656666313564666563653838333338616465623734 +61333761373630363937663463653464396532343065616235333866346530623137393736323263 +33366664616666616336653636346364643331646461616265643036613933303661613738633164 +61633261363932323533346133386337393339363139323263633261666130623265363765323263 +62616533346262666431353435353836356361663339393637363337613566333633326337636566 +36623734326631623463306431616639656438323062363236613930333561343765353066646230 +32666337373730376536363563623962636238663035383932666265633839303832323035626137 +33343433666261303461373330323137646435306533353733323361353363616561343437383263 +61373666376564306232613661303666373363346237356234313261343666623462656461623564 +32663566666562383435346563633862353135336330386533343431626161656463356635353335 +35353630393538663136333261623766616539666663353230316531373664373030643265333539 +38393536353865653566326334313332313365636261303635626465346330656235343165656338 +35313030356434643363373138393665333633333931653639336438663066656437303565303439 +33386630346361383532393663623739303038636432326132343263316137626234373933386231 +32396362313562346234303466626562373566313466633837353534333530616632663633303264 +31343766623231313035383263306463653162333131616331663133633062393438323961353435 +61323838373930353136393564396533323966376534316361393261613339633762623934306162 +63313361376130363863353865336635386137383561343039643661623866643037656131663737 +37646337633438393534326639343637623133326138376637623065313831313537623534623235 +61663234393139323137323831306132326334383333623330666364313530353732393163666431 +38353765353762633535613938353533633934326533333332346633343165336632393239666438 +30373532653162366539306435356535666331643839663565643436336238333235303236343138 +38666232633366316631303037383330613032333863663339333931343563383836346632646665 +37393230336437366430393931383163666136376463333465363838376338653064636334663238 +62366632653861316536343035633732313333636231666162393735333334323465393331333166 +31613831326630333366643237623263613131376436666534643131613939393666306536656464 +64396461613032653866323466303562656465643866353736383535643930613930656564333431 +32633831643164326466653963633036663862656431386133653837643430343934323066303263 +31333731666466663364333135653037636131636230393331626166616337623564363339613861 +36336434663633323632316264306239373636306531303266353235393039316366376665616566 +32373462666565616332376337653933666639346236643234626336356664356430626134373132 +38663031303433356533356330646161376561373936626537616232643166303133353565663336 +30643239363933613339643938356532616539643636653663666366653332626134346463356261 +62303362346535373333303566666435373163343931643762333630363062393830636339386435 +33646239386234353536303434303964376534313134323236333235623535333031326538653661 +30326166316539373534313863636231643630636535366566333632373536623663373239623633 +61306236346236343535373461303762633564636238616437326233643961633538316530313263 +66666332633165613933613064303436363632663435333535393962323337303661323266376532 +31633963353031393663363366366130623462643630386232336366306432373733343061323737 +66316262363135636534646665613333663031613032303133656334663165383537633166373835 +38306365393863663566646231373035613037633533343266353664373033376161323437646637 +30386161656138333864646262326536326161653730333232353234306230663436313635333234 +30336239643065613561393333313131666138343533636562653965393566666265313835393665 +33333733613130366436333432343435316266633335356565303935633261333462393532373866 +34303731643838663433646634623935636165353533613834306561633935656265373666643666 +30383364333436306538373566663566343966363565313033363738646338343563393562373235 +62366235336461313232316539636262363566376363383664313563396336623763656639653964 +63356163653736623466386536353830623761636535346338396134616362346337346462373938 +64643162303163373062313039346433346532356136353065353663666666336337313362653133 +36356433623361313034313936303264633366643530323635613761383965383361343465326364 +39393062396430363336626362366430633331366461343930646662326565333661623336303232 +36353065313237303532623563646230646139613334336639353163646461353065616464313132 +63663830336636326336663363383531333765653462363566383731353063303439653930636165 +64386536343932613966346161366266323230663531373765393065656465636135363330653766 +3030 diff --git a/ansible/inventory/host_vars/ariel.incus.yml b/ansible/inventory/host_vars/ariel.incus.yml index 573c3ab..3be6929 100644 --- a/ansible/inventory/host_vars/ariel.incus.yml +++ b/ansible/inventory/host_vars/ariel.incus.yml @@ -15,8 +15,7 @@ neo4j_syslog_port: 51414 neo4j_user: neo4j neo4j_group: neo4j neo4j_directory: /srv/neo4j -neo4j_auth_user: neo4j -neo4j_auth_password: "{{ vault_neo4j_auth_password }}" +neo4j_password: "{{ vault_neo4j_cypher_password }}" neo4j_http_port: 22084 neo4j_bolt_port: 22074 neo4j_metrics_port: 22094 diff --git a/ansible/inventory/host_vars/miranda.incus.yml b/ansible/inventory/host_vars/miranda.incus.yml index 8a1aea2..3facceb 100644 --- a/ansible/inventory/host_vars/miranda.incus.yml +++ b/ansible/inventory/host_vars/miranda.incus.yml @@ -10,21 +10,23 @@ services: - grafana_mcp - mcpo - neo4j_mcp + - searxng # Alloy alloy_log_level: "warn" -argos_syslog_port: 51434 -neo4j_cypher_syslog_port: 51431 -grafana_mcp_syslog_port: 51433 -gitea_mcp_syslog_port: 51435 +argos_syslog_port: 51418 +neo4j_cypher_syslog_port: 51414 +grafana_mcp_syslog_port: 51413 +gitea_mcp_syslog_port: 51412 +searxng_syslog_port: 51419 # Argos MCP Configuration argos_user: argos argos_group: argos argos_directory: /srv/argos -argos_port: 25534 +argos_port: 20861 argos_log_level: INFO -argos_searxng_instances: http://rosalind.incus:22089/ +argos_searxng_instances: http://miranda.incus:22089/,http://rosalind.incus:22089/ argos_cache_ttl: 300 argos_max_results: 10 argos_request_timeout: 30.0 @@ -48,7 +50,7 @@ neo4j_mcp_directory: /srv/neo4j_mcp grafana_mcp_user: grafana_mcp grafana_mcp_group: grafana_mcp grafana_mcp_directory: /srv/grafana_mcp -grafana_mcp_port: 25533 +grafana_mcp_port: 22063 grafana_mcp_grafana_host: prospero.incus grafana_mcp_grafana_port: 3000 grafana_service_account_token: "{{ vault_grafana_service_account_token }}" @@ -57,21 +59,28 @@ grafana_service_account_token: "{{ vault_grafana_service_account_token }}" gitea_mcp_user: gitea_mcp gitea_mcp_group: gitea_mcp gitea_mcp_directory: /srv/gitea_mcp -gitea_mcp_port: 25535 +gitea_mcp_port: 22062 gitea_mcp_host: https://gitea.ouranos.helu.ca gitea_mcp_access_token: "{{ vault_gitea_mcp_access_token }}" # Neo4j Cypher MCP neo4j_host: ariel.incus -neo4j_bolt_port: 7687 -neo4j_auth_password: "{{ vault_neo4j_auth_password }}" -neo4j_cypher_mcp_port: 25531 - -# Nike MCP -nike_mcp_url: http://puck.incus:25576/mcp +neo4j_bolt_port: 22074 +neo4j_cypher_password: "{{ vault_neo4j_cypher_password }}" +neo4j_cypher_mcp_port: 22064 +neo4j_mcp_server_allowed_hosts: localhost,127.0.0.1,miranda.incus # MCPO Config mcpo_user: mcpo mcpo_group: mcpo mcpo_directory: /srv/mcpo mcpo_port: 25530 + +# SearXNG Configuration +searxng_user: searxng +searxng_group: searxng +searxng_directory: /srv/searxng +searxng_port: 22089 +searxng_base_url: http://miranda.incus:22089/ +searxng_instance_name: "Ouranos Search" +searxng_secret_key: "{{ vault_searxng_secret_key }}" diff --git a/ansible/inventory/host_vars/prospero.incus.yml b/ansible/inventory/host_vars/prospero.incus.yml index f407559..e78d751 100644 --- a/ansible/inventory/host_vars/prospero.incus.yml +++ b/ansible/inventory/host_vars/prospero.incus.yml @@ -74,6 +74,22 @@ prometheus_targets: - 'rosalind.incus:9100' - 'umbriel.incus:9100' +# Neo4j scrape targets (neo4j-apoc-exporter sidecar on each Neo4j host) +neo4j_metrics_targets: + - 'ariel.incus:22094' + - 'umbriel.incus:22094' + +# Pallas scrape targets — one entry per Pallas deployment (registry +# port). The `instance` label distinguishes deployments; the `agent` +# dimension comes from labels on the metrics themselves. +pallas_metrics_targets: + - targets: ['caliban.incus:24000'] + labels: {instance: iolaus} + - targets: ['caliban.incus:24100'] + labels: {instance: kottos} + - targets: ['caliban.incus:24200'] + labels: {instance: mentor} + # Prometheus OAuth2-Proxy Sidecar prometheus_proxy_port: 9091 prometheus_oauth2_proxy_dir: /etc/oauth2-proxy-prometheus @@ -127,10 +143,23 @@ pgadmin_oauth_client_id: "{{ vault_pgadmin_oauth_client_id }}" pgadmin_oauth_client_secret: "{{ vault_pgadmin_oauth_client_secret }}" # ============================================================================ -# Casdoor Metrics (for Prometheus scraping) +# Prometheus Metrics Scraping # ============================================================================ -casdoor_metrics_host: "titania.incus" +# Casdoor +casdoor_metrics_host: titania.incus casdoor_metrics_port: 22081 casdoor_prometheus_access_key: "{{ vault_casdoor_prometheus_access_key }}" casdoor_prometheus_access_secret: "{{ vault_casdoor_prometheus_access_secret }}" + +# Daedalus Metrics +daedalus_metrics_host: caliban.incus +daedalus_metrics_port: 23081 + +# Mnemosyne — two scrape targets: +# app: Django /metrics via nginx (django-prometheus + custom pipeline/MCP counters) +# web: nginx-prometheus-exporter sidecar (nginx stub_status → Prometheus format) +mnemosyne_app_metrics_host: caliban.incus +mnemosyne_app_metrics_port: 23181 +mnemosyne_web_metrics_host: caliban.incus +mnemosyne_web_metrics_port: 23191 diff --git a/ansible/inventory/host_vars/puck.incus.yml b/ansible/inventory/host_vars/puck.incus.yml index a15e3e3..862f536 100644 --- a/ansible/inventory/host_vars/puck.incus.yml +++ b/ansible/inventory/host_vars/puck.incus.yml @@ -37,14 +37,17 @@ daedalus_syslog_port: 51430 # Daedalus (FastAPI on puck, behind nginx) daedalus_metrics_host: "puck.incus" -daedalus_metrics_port: 22181 +daedalus_metrics_port: 23081 -# Mnemosyne — /metrics is served by nginx (mnemosyne-web:23181) and -# proxied to the Django app container, which owns the single -# prometheus_client process registry that both django-prometheus -# (HTTP / Celery) and the MCP server's tool-call counters write to. -mnemosyne_metrics_host: "puck.incus" -mnemosyne_metrics_port: 23181 +# Mnemosyne — two metrics surfaces: +# app (23181): /metrics served by nginx → Django app container, which owns +# the single prometheus_client process registry that both django-prometheus +# (HTTP / Celery) and the MCP server's tool-call counters write to. +# web (23191): nginx-prometheus-exporter sidecar scraping nginx stub_status. +mnemosyne_app_metrics_host: "puck.incus" +mnemosyne_app_metrics_port: 23181 +mnemosyne_web_metrics_host: "puck.incus" +mnemosyne_web_metrics_port: 23191 # ============================================================================= # Kottos Configuration (Pallas FastAgent runtime) diff --git a/ansible/inventory/host_vars/titania.incus.yml b/ansible/inventory/host_vars/titania.incus.yml index 1cfa404..86427a4 100644 --- a/ansible/inventory/host_vars/titania.incus.yml +++ b/ansible/inventory/host_vars/titania.incus.yml @@ -122,8 +122,8 @@ haproxy_backends: health_path: "/api/healthz" - subdomain: "daedalus" - backend_host: "puck.incus" - backend_port: 20080 + backend_host: "caliban.incus" + backend_port: 20081 health_path: "/ready/" timeout_server: 120s @@ -133,8 +133,8 @@ haproxy_backends: health_path: "/chat" - subdomain: "mnemosyne" - backend_host: "puck.incus" - backend_port: 23181 + backend_host: "caliban.incus" + backend_port: 23081 health_path: "/ready/" - subdomain: "nextcloud" diff --git a/ansible/inventory/host_vars/umbriel.incus.yml b/ansible/inventory/host_vars/umbriel.incus.yml index ee84f69..858881a 100644 --- a/ansible/inventory/host_vars/umbriel.incus.yml +++ b/ansible/inventory/host_vars/umbriel.incus.yml @@ -19,8 +19,7 @@ neo4j_syslog_port: 51414 neo4j_user: neo4j neo4j_group: neo4j neo4j_directory: /srv/neo4j -neo4j_auth_user: neo4j -neo4j_auth_password: "{{ vault_mnemosyne_neo4j_auth_password }}" +neo4j_password: "{{ vault_neo4j_mnemosyne_password }}" neo4j_http_port: 22084 neo4j_bolt_port: 22074 neo4j_metrics_port: 22094 diff --git a/ansible/mcpo/config.json.j2 b/ansible/mcpo/config.json.j2 index 38fed71..45ef5c2 100644 --- a/ansible/mcpo/config.json.j2 +++ b/ansible/mcpo/config.json.j2 @@ -4,47 +4,17 @@ "command": "/srv/mcpo/.venv/bin/python", "args": ["/srv/mcpo/.venv/bin/mcp-server-time", "--local-timezone=America/Toronto"] }, - "upstash-context7": { - "command": "npx", - "args": [ - "-y", - "@upstash/context7-mcp" - ] - }, - "angelia": { - "url": "https://ouranos.helu.ca/mcp/sse/", - "headers": { - "Authorization": "Bearer LmDTU1OoQm7nk8-T7NtGwwA5aut7LqcpVYpLxRKUS51klljJkFUbmu3KYnR8V6Ww" - } - }, "argos": { "type": "streamable_http", "url": "{{argos_mcp_url}}" }, - "athena": { - "url": "https://athena.ouranos.helu.ca/mcp/sse/", - "headers": { - "Authorization": "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIyIiwidXNlcl9pZCI6MiwidXNlcm5hbWUiOiJyQGhlbHUuY2EiLCJpc3MiOiJhdGhlbmEiLCJhdWQiOiJhdGhlbmEtbWNwIiwiaWF0IjoxNzczODc4MDgwLCJrZXlfbmFtZSI6Ik1pcmFuZGEgTUNQTyBLZXkiLCJ0ZW5hbnRfaWQiOjF9.bpFKRbfEygKOW6_UlfQ7H5ZZZ5-LgMJ2UP653GhpZ5A" - }, - "caliban": { - "type": "streamable_http", - "url": "{{caliban_mcp_url}}" - }, "gitea": { "type": "streamable_http", "url": "{{gitea_mcp_url}}" }, - "korax": { - "type": "streamable_http", - "url": "{{korax_mcp_url}}" - }, "neo4j-cypher": { "type": "streamable_http", "url": "{{neo4j_mcp_url}}" - }, - "nike": { - "type": "streamable_http", - "url": "{{nike_mcp_url}}" } } } diff --git a/ansible/neo4j/docker-compose.yml.j2 b/ansible/neo4j/docker-compose.yml.j2 index 2db6f3b..cc81578 100644 --- a/ansible/neo4j/docker-compose.yml.j2 +++ b/ansible/neo4j/docker-compose.yml.j2 @@ -21,6 +21,7 @@ services: NEO4J_apoc_import_file_enabled: "true" NEO4J_apoc_import_file_use__neo4j__config: "true" NEO4J_dbms_security_procedures_unrestricted: "{{neo4j_apoc_unrestricted}}" + NEO4J_server_default__listen__address: "0.0.0.0" logging: driver: syslog options: diff --git a/ansible/neo4j_mcp/docker-compose.yml.j2 b/ansible/neo4j_mcp/docker-compose.yml.j2 index a983dee..de84760 100644 --- a/ansible/neo4j_mcp/docker-compose.yml.j2 +++ b/ansible/neo4j_mcp/docker-compose.yml.j2 @@ -1,7 +1,7 @@ +# Generated by Ansible - do not edit manually services: neo4j-cypher: - image: mcp/neo4j-cypher:latest - pull_policy: always + image: mcp/neo4j-cypher:{{ neo4j_mcp_image_version }} container_name: neo4j-cypher restart: unless-stopped ports: @@ -9,14 +9,14 @@ services: environment: - NEO4J_URI=bolt://{{neo4j_host}}:{{neo4j_bolt_port}} - NEO4J_USERNAME=neo4j - - NEO4J_PASSWORD={{neo4j_auth_password}} + - NEO4J_PASSWORD={{neo4j_cypher_password}} - NEO4J_DATABASE=neo4j - NEO4J_TRANSPORT=http - NEO4J_MCP_SERVER_HOST=0.0.0.0 - NEO4J_MCP_SERVER_PORT=8000 - NEO4J_MCP_SERVER_PATH=/mcp - NEO4J_NAMESPACE=local - - NEO4J_MCP_SERVER_ALLOWED_HOSTS=localhost,127.0.0.1,miranda.incus,rosalind.incus,miranda.incus:{{neo4j_cypher_mcp_port}} + - NEO4J_MCP_SERVER_ALLOWED_HOSTS={{neo4j_mcp_server_allowed_hosts}} - NEO4J_MCP_SERVER_ALLOW_ORIGINS= - NEO4J_READ_TIMEOUT=30 logging: diff --git a/ansible/pplg/dashboards_provider.yml.j2 b/ansible/pplg/dashboards_provider.yml.j2 deleted file mode 100644 index d1e67b5..0000000 --- a/ansible/pplg/dashboards_provider.yml.j2 +++ /dev/null @@ -1,23 +0,0 @@ -# Grafana dashboard file provider -# Deployed to: /etc/grafana/provisioning/dashboards/puck.yaml -# -# Grafana polls the ``path`` every ``updateIntervalSeconds`` and re-imports -# any JSON file it finds. Each dashboard JSON lives in that directory and -# is owned by Ansible — operators should not edit dashboards through the -# Grafana UI (changes won't survive a deploy; export the final JSON and -# land it in this role). -apiVersion: 1 - -providers: - - name: 'puck' - orgId: 1 - folder: 'Puck Services' - folderUid: puck-services - type: file - disableDeletion: false - editable: true - allowUiUpdates: false - updateIntervalSeconds: 30 - options: - path: /var/lib/grafana/dashboards - foldersFromFilesStructure: false diff --git a/ansible/pplg/deploy.yml b/ansible/pplg/deploy.yml index bc718b4..49f496f 100644 --- a/ansible/pplg/deploy.yml +++ b/ansible/pplg/deploy.yml @@ -200,40 +200,6 @@ # Grafana # =========================================================================== - - name: Create dashboards directory - ansible.builtin.file: - path: /var/lib/grafana/dashboards - state: directory - owner: grafana - group: grafana - mode: '750' - - - name: Ensure Grafana dashboard provisioning directory exists - ansible.builtin.file: - path: /etc/grafana/provisioning/dashboards - state: directory - owner: grafana - group: grafana - mode: '750' - - - name: Template Grafana dashboard provider (file source → /var/lib/grafana/dashboards) - ansible.builtin.template: - src: "dashboards_provider.yml.j2" - dest: "/etc/grafana/provisioning/dashboards/puck.yaml" - owner: grafana - group: grafana - mode: '640' - notify: restart grafana - - - name: Template Puck Services dashboard (Mnemosyne + Pallas + Daedalus) - ansible.builtin.template: - src: "puck_services_dashboard.json.j2" - dest: "/var/lib/grafana/dashboards/puck_services.json" - owner: grafana - group: grafana - mode: '640' - notify: restart grafana - - name: Template Grafana main configuration ansible.builtin.template: src: "grafana.ini.j2" diff --git a/ansible/pplg/prometheus.yml.j2 b/ansible/pplg/prometheus.yml.j2 index 4b9b72b..3b8eaee 100644 --- a/ansible/pplg/prometheus.yml.j2 +++ b/ansible/pplg/prometheus.yml.j2 @@ -51,28 +51,44 @@ scrape_configs: metrics_path: '/metrics' scrape_interval: 15s - # Mnemosyne — single /metrics endpoint on the app container serves both - # django-prometheus HTTP/Celery metrics and the MCP server's tool-call - # counters (the mcp_server.metrics module registers into the same - # prometheus_client process registry on the Django side). The mcp - # container itself does not expose /metrics; run 'em on the WSGI side. + # Mnemosyne — app exposes /metrics on the Django container (proxied via + # nginx); a single prometheus_client process registry serves both + # django-prometheus (HTTP/Celery) and the MCP server's tool-call counters + # (the mcp container itself does not expose /metrics). Web is an + # nginx-prometheus-exporter sidecar that scrapes the web container's + # stub_status and re-exposes it in Prometheus format. - job_name: 'mnemosyne' - static_configs: - - targets: ['{{ mnemosyne_metrics_host }}:{{ mnemosyne_metrics_port }}'] metrics_path: '/metrics' scrape_interval: 15s + static_configs: + - targets: ['{{ mnemosyne_app_metrics_host }}:{{ mnemosyne_app_metrics_port }}'] + labels: + component: app + - targets: ['{{ mnemosyne_web_metrics_host }}:{{ mnemosyne_web_metrics_port }}'] + labels: + component: web + + # Pallas — each deployment is one scrape target (registry port). + # Pallas uses a single process-global registry, so per-agent /metrics + # endpoints serve the same snapshot; the `agent` dimension is carried + # as a metric label, not a target. Targets are defined per + # environment in pallas_metrics_targets (host_vars on the Prometheus + # host); instances are differentiated by the `instance` label. +{% if pallas_metrics_targets | default([]) %} + - job_name: 'pallas' + metrics_path: '/metrics' + scrape_interval: 15s + static_configs: {{ pallas_metrics_targets | to_json }} +{% endif %} # Neo4j — stscoundrel/neo4j-apoc-exporter sidecar connects to the local # Neo4j over Bolt and exposes apoc.monitor.* (tx/ids/store) plus JVM - # metrics on the standard metrics port (22094). Both Ariel (LLM memory - # via neo4j-cypher MCP) and Umbriel (Mnemosyne graph+vector DB) use the - # same port — they are differentiated by hostname only. + # metrics. Targets are listed per-environment in neo4j_metrics_targets + # (host_vars on the Prometheus host) — instances are differentiated by + # hostname only. - job_name: 'neo4j' static_configs: - - targets: - - 'ariel.incus:22094' - - 'umbriel.incus:22094' + - targets: {{ neo4j_metrics_targets | to_json }} metrics_path: '/metrics' scrape_interval: 15s -# Red Panda Approved Prometheus Configuration diff --git a/ansible/pplg/puck_services_dashboard.json.j2 b/ansible/pplg/puck_services_dashboard.json.j2 deleted file mode 100644 index f17d3f5..0000000 --- a/ansible/pplg/puck_services_dashboard.json.j2 +++ /dev/null @@ -1,242 +0,0 @@ -{ - "title": "Puck Services — Logs & Health", - "uid": "puck-services-logs", - "tags": ["puck", "logs", "mnemosyne", "pallas", "daedalus"], - "timezone": "browser", - "schemaVersion": 39, - "version": 1, - "editable": true, - "fiscalYearStartMonth": 0, - "weekStart": "", - "refresh": "30s", - "time": {"from": "now-1h", "to": "now"}, - "templating": { - "list": [ - { - "name": "loki", - "type": "datasource", - "query": "loki", - "current": {"selected": false, "text": "Loki", "value": "Loki"}, - "hide": 0, - "label": "Loki datasource" - }, - { - "name": "prom", - "type": "datasource", - "query": "prometheus", - "current": {"selected": false, "text": "Prometheus", "value": "Prometheus"}, - "hide": 0, - "label": "Prometheus datasource" - } - ] - }, - "panels": [ - { - "id": 1, - "type": "row", - "title": "Mnemosyne", - "collapsed": false, - "gridPos": {"h": 1, "w": 24, "x": 0, "y": 0} - }, - { - "id": 2, - "type": "timeseries", - "title": "Mnemosyne — log rate by level", - "datasource": {"type": "loki", "uid": "${loki}"}, - "gridPos": {"h": 8, "w": 12, "x": 0, "y": 1}, - "targets": [ - { - "refId": "A", - "expr": "sum by (level) (rate({service=\"mnemosyne\"} | json [5m]))", - "legendFormat": "{{level}}" - } - ], - "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}} - }, - { - "id": 3, - "type": "logs", - "title": "Mnemosyne — errors (last 25)", - "datasource": {"type": "loki", "uid": "${loki}"}, - "gridPos": {"h": 8, "w": 12, "x": 12, "y": 1}, - "targets": [ - { - "refId": "A", - "expr": "{service=\"mnemosyne\"} | json | level=\"ERROR\"", - "maxLines": 25 - } - ], - "options": {"showLabels": false, "showTime": true, "wrapLogMessage": true} - }, - { - "id": 4, - "type": "stat", - "title": "Mnemosyne — HTTP 5xx rate", - "datasource": {"type": "prometheus", "uid": "${prom}"}, - "gridPos": {"h": 4, "w": 8, "x": 0, "y": 9}, - "targets": [ - { - "refId": "A", - "expr": "sum(rate(django_http_responses_total_by_status_total{job=\"mnemosyne\",status=~\"5..\"}[5m])) / clamp_min(sum(rate(django_http_responses_total_by_status_total{job=\"mnemosyne\"}[5m])), 0.0001)" - } - ], - "options": { - "reduceOptions": {"calcs": ["lastNotNull"]}, - "colorMode": "value", - "textMode": "auto" - }, - "fieldConfig": {"defaults": {"unit": "percentunit", "thresholds": {"mode": "absolute", "steps": [{"color": "green"}, {"color": "orange", "value": 0.01}, {"color": "red", "value": 0.05}]}}} - }, - { - "id": 5, - "type": "stat", - "title": "Mnemosyne — p95 latency", - "datasource": {"type": "prometheus", "uid": "${prom}"}, - "gridPos": {"h": 4, "w": 8, "x": 8, "y": 9}, - "targets": [ - { - "refId": "A", - "expr": "histogram_quantile(0.95, sum by (le) (rate(django_http_requests_latency_including_middlewares_seconds_bucket{job=\"mnemosyne\"}[5m])))" - } - ], - "options": {"reduceOptions": {"calcs": ["lastNotNull"]}}, - "fieldConfig": {"defaults": {"unit": "s", "thresholds": {"mode": "absolute", "steps": [{"color": "green"}, {"color": "orange", "value": 1}, {"color": "red", "value": 5}]}}} - }, - { - "id": 6, - "type": "stat", - "title": "Mnemosyne — MCP tool error rate", - "datasource": {"type": "prometheus", "uid": "${prom}"}, - "gridPos": {"h": 4, "w": 8, "x": 16, "y": 9}, - "targets": [ - { - "refId": "A", - "expr": "sum(rate(mcp_tool_invocations_total{job=\"mnemosyne\",status=\"error\"}[5m])) / clamp_min(sum(rate(mcp_tool_invocations_total{job=\"mnemosyne\"}[5m])), 0.0001)" - } - ], - "options": {"reduceOptions": {"calcs": ["lastNotNull"]}}, - "fieldConfig": {"defaults": {"unit": "percentunit", "thresholds": {"mode": "absolute", "steps": [{"color": "green"}, {"color": "orange", "value": 0.05}, {"color": "red", "value": 0.10}]}}} - }, - - { - "id": 10, - "type": "row", - "title": "Pallas (Kottos agents)", - "collapsed": false, - "gridPos": {"h": 1, "w": 24, "x": 0, "y": 13} - }, - { - "id": 11, - "type": "timeseries", - "title": "Pallas — log rate by agent (component)", - "datasource": {"type": "loki", "uid": "${loki}"}, - "gridPos": {"h": 8, "w": 12, "x": 0, "y": 14}, - "targets": [ - { - "refId": "A", - "expr": "sum by (component) (rate({service=\"pallas\", project=\"kottos\"} | json [5m]))", - "legendFormat": "{{component}}" - } - ], - "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}} - }, - { - "id": 12, - "type": "logs", - "title": "Pallas — forward trace errors (opaque MCP transport failures)", - "datasource": {"type": "loki", "uid": "${loki}"}, - "gridPos": {"h": 8, "w": 12, "x": 12, "y": 14}, - "targets": [ - { - "refId": "A", - "expr": "{service=\"pallas\", project=\"kottos\"} |= \"pallas.forward.trace\" | json | level=~\"ERROR|WARNING\"", - "maxLines": 25 - } - ], - "options": {"showLabels": false, "showTime": true, "wrapLogMessage": true} - }, - { - "id": 13, - "type": "logs", - "title": "Pallas — last 25 ERROR lines (any agent)", - "datasource": {"type": "loki", "uid": "${loki}"}, - "gridPos": {"h": 8, "w": 24, "x": 0, "y": 22}, - "targets": [ - { - "refId": "A", - "expr": "{service=\"pallas\", project=\"kottos\"} | json | level=\"ERROR\"", - "maxLines": 25 - } - ], - "options": {"showLabels": true, "showTime": true, "wrapLogMessage": true} - }, - - { - "id": 20, - "type": "row", - "title": "Daedalus", - "collapsed": false, - "gridPos": {"h": 1, "w": 24, "x": 0, "y": 30} - }, - { - "id": 21, - "type": "timeseries", - "title": "Daedalus — log rate by level", - "datasource": {"type": "loki", "uid": "${loki}"}, - "gridPos": {"h": 8, "w": 12, "x": 0, "y": 31}, - "targets": [ - { - "refId": "A", - "expr": "sum by (level) (rate({service=\"daedalus\"} | json [5m]))", - "legendFormat": "{{level}}" - } - ], - "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}} - }, - { - "id": 22, - "type": "stat", - "title": "Daedalus — HTTP 5xx rate", - "datasource": {"type": "prometheus", "uid": "${prom}"}, - "gridPos": {"h": 4, "w": 6, "x": 12, "y": 31}, - "targets": [ - { - "refId": "A", - "expr": "sum(rate(daedalus_http_requests_total{status=~\"5..\"}[5m])) / clamp_min(sum(rate(daedalus_http_requests_total[5m])), 0.0001)" - } - ], - "options": {"reduceOptions": {"calcs": ["lastNotNull"]}}, - "fieldConfig": {"defaults": {"unit": "percentunit", "thresholds": {"mode": "absolute", "steps": [{"color": "green"}, {"color": "orange", "value": 0.01}, {"color": "red", "value": 0.05}]}}} - }, - { - "id": 23, - "type": "stat", - "title": "Daedalus — MCP p95 latency", - "datasource": {"type": "prometheus", "uid": "${prom}"}, - "gridPos": {"h": 4, "w": 6, "x": 18, "y": 31}, - "targets": [ - { - "refId": "A", - "expr": "histogram_quantile(0.95, rate(daedalus_mcp_request_duration_seconds_bucket[5m]))" - } - ], - "options": {"reduceOptions": {"calcs": ["lastNotNull"]}}, - "fieldConfig": {"defaults": {"unit": "s", "thresholds": {"mode": "absolute", "steps": [{"color": "green"}, {"color": "orange", "value": 5}, {"color": "red", "value": 30}]}}} - }, - { - "id": 24, - "type": "logs", - "title": "Daedalus — errors (last 25)", - "datasource": {"type": "loki", "uid": "${loki}"}, - "gridPos": {"h": 8, "w": 24, "x": 0, "y": 39}, - "targets": [ - { - "refId": "A", - "expr": "{service=\"daedalus\"} | json | level=\"ERROR\"", - "maxLines": 25 - } - ], - "options": {"showLabels": false, "showTime": true, "wrapLogMessage": true} - } - ] -} diff --git a/ansible/searxng/deploy.yml b/ansible/searxng/deploy.yml index 4fe6423..926590c 100644 --- a/ansible/searxng/deploy.yml +++ b/ansible/searxng/deploy.yml @@ -57,78 +57,3 @@ project_src: "{{searxng_directory}}" state: present pull: always - - # =========================================================================== - # OAuth2-Proxy Sidecar - # Note: Each host supports at most one OAuth2-Proxy sidecar instance - # (binary shared at /usr/local/bin/oauth2-proxy, unique systemd unit per service) - # =========================================================================== - - name: Create oauth2-proxy directory - ansible.builtin.file: - path: "{{ searxng_oauth2_proxy_dir }}" - owner: root - group: root - state: directory - mode: '0755' - - - name: Download oauth2-proxy binary - ansible.builtin.get_url: - url: "https://github.com/oauth2-proxy/oauth2-proxy/releases/download/v{{ searxng_oauth2_proxy_version }}/oauth2-proxy-v{{ searxng_oauth2_proxy_version }}.linux-amd64.tar.gz" - dest: "/tmp/oauth2-proxy-v{{ searxng_oauth2_proxy_version }}.tar.gz" - mode: '0644' - - - name: Extract oauth2-proxy binary - ansible.builtin.unarchive: - src: "/tmp/oauth2-proxy-v{{ searxng_oauth2_proxy_version }}.tar.gz" - dest: /tmp - remote_src: true - creates: "/tmp/oauth2-proxy-v{{ searxng_oauth2_proxy_version }}.linux-amd64/oauth2-proxy" - - - name: Install oauth2-proxy binary - ansible.builtin.copy: - src: "/tmp/oauth2-proxy-v{{ searxng_oauth2_proxy_version }}.linux-amd64/oauth2-proxy" - dest: /usr/local/bin/oauth2-proxy - owner: root - group: root - mode: '0755' - remote_src: true - - - name: Template oauth2-proxy configuration - ansible.builtin.template: - src: oauth2-proxy-searxng.cfg.j2 - dest: "{{ searxng_oauth2_proxy_dir }}/oauth2-proxy.cfg" - owner: root - group: root - mode: '0600' - notify: restart oauth2-proxy-searxng - - - name: Template oauth2-proxy systemd service - ansible.builtin.template: - src: oauth2-proxy-searxng.service.j2 - dest: /etc/systemd/system/oauth2-proxy-searxng.service - owner: root - group: root - mode: '0644' - notify: - - reload systemd - - restart oauth2-proxy-searxng - - # =========================================================================== - # Service Management - # =========================================================================== - - name: Enable and start OAuth2-Proxy service - ansible.builtin.systemd: - name: oauth2-proxy-searxng - enabled: true - state: started - daemon_reload: true - - handlers: - - name: reload systemd - ansible.builtin.systemd: - daemon_reload: true - - - name: restart oauth2-proxy-searxng - ansible.builtin.systemd: - name: oauth2-proxy-searxng - state: restarted diff --git a/ansible/searxng/deploy_oauth2.yml b/ansible/searxng/deploy_oauth2.yml new file mode 100644 index 0000000..c50e465 --- /dev/null +++ b/ansible/searxng/deploy_oauth2.yml @@ -0,0 +1,86 @@ +--- +- name: Deploy OAuth2-Proxy sidecar for SearXNG + hosts: ubuntu + become: true + tasks: + - name: Check if host has searxng service with OAuth2 configured + ansible.builtin.set_fact: + has_searxng_oauth2: >- + {{ 'searxng' in services + and (searxng_oauth2_client_id | default('')) | length > 0 }} + + - name: Skip hosts without SearXNG OAuth2-Proxy configuration + ansible.builtin.meta: end_host + when: not has_searxng_oauth2 + + # =========================================================================== + # OAuth2-Proxy Sidecar + # Note: Each host supports at most one OAuth2-Proxy sidecar instance + # (binary shared at /usr/local/bin/oauth2-proxy, unique systemd unit per service) + # =========================================================================== + - name: Create oauth2-proxy directory + ansible.builtin.file: + path: "{{ searxng_oauth2_proxy_dir }}" + owner: root + group: root + state: directory + mode: '0755' + + - name: Download oauth2-proxy binary + ansible.builtin.get_url: + url: "https://github.com/oauth2-proxy/oauth2-proxy/releases/download/v{{ searxng_oauth2_proxy_version }}/oauth2-proxy-v{{ searxng_oauth2_proxy_version }}.linux-amd64.tar.gz" + dest: "/tmp/oauth2-proxy-v{{ searxng_oauth2_proxy_version }}.tar.gz" + mode: '0644' + + - name: Extract oauth2-proxy binary + ansible.builtin.unarchive: + src: "/tmp/oauth2-proxy-v{{ searxng_oauth2_proxy_version }}.tar.gz" + dest: /tmp + remote_src: true + creates: "/tmp/oauth2-proxy-v{{ searxng_oauth2_proxy_version }}.linux-amd64/oauth2-proxy" + + - name: Install oauth2-proxy binary + ansible.builtin.copy: + src: "/tmp/oauth2-proxy-v{{ searxng_oauth2_proxy_version }}.linux-amd64/oauth2-proxy" + dest: /usr/local/bin/oauth2-proxy + owner: root + group: root + mode: '0755' + remote_src: true + + - name: Template oauth2-proxy configuration + ansible.builtin.template: + src: oauth2-proxy-searxng.cfg.j2 + dest: "{{ searxng_oauth2_proxy_dir }}/oauth2-proxy.cfg" + owner: root + group: root + mode: '0600' + notify: restart oauth2-proxy-searxng + + - name: Template oauth2-proxy systemd service + ansible.builtin.template: + src: oauth2-proxy-searxng.service.j2 + dest: /etc/systemd/system/oauth2-proxy-searxng.service + owner: root + group: root + mode: '0644' + notify: + - reload systemd + - restart oauth2-proxy-searxng + + - name: Enable and start OAuth2-Proxy service + ansible.builtin.systemd: + name: oauth2-proxy-searxng + enabled: true + state: started + daemon_reload: true + + handlers: + - name: reload systemd + ansible.builtin.systemd: + daemon_reload: true + + - name: restart oauth2-proxy-searxng + ansible.builtin.systemd: + name: oauth2-proxy-searxng + state: restarted diff --git a/ansible/site.yml b/ansible/site.yml index d7bda5f..c553150 100644 --- a/ansible/site.yml +++ b/ansible/site.yml @@ -33,6 +33,9 @@ - name: Deploy SearXNG import_playbook: searxng/deploy.yml +- name: Deploy SearXNG OAuth2-Proxy sidecar + import_playbook: searxng/deploy_oauth2.yml + - name: Deploy HAProxy import_playbook: haproxy/deploy.yml diff --git a/dashboards/argos.json b/dashboards/argos.json new file mode 100644 index 0000000..cc762e7 --- /dev/null +++ b/dashboards/argos.json @@ -0,0 +1,307 @@ +{ + "title": "Argos", + "uid": "argos", + "tags": ["argos", "mcp", "searxng", "ouranos"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": true, + "fiscalYearStartMonth": 0, + "weekStart": "", + "refresh": "30s", + "time": {"from": "now-1h", "to": "now"}, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "SearXNG dashboard", + "tooltip": "SearXNG instance probes (miranda, rosalind)", + "type": "link", + "url": "/d/searxng" + } + ], + "templating": { + "list": [ + { + "name": "prom", + "type": "datasource", + "query": "prometheus", + "current": {"selected": false, "text": "Prometheus", "value": "Prometheus"}, + "hide": 0, + "label": "Prometheus datasource" + }, + { + "name": "loki", + "type": "datasource", + "query": "loki", + "current": {"selected": false, "text": "Loki", "value": "Loki"}, + "hide": 0, + "label": "Loki datasource" + }, + { + "name": "instance", + "type": "query", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "query": "label_values(up{job=\"argos\"}, instance)", + "refresh": 1, + "includeAll": true, + "multi": true, + "current": {"selected": true, "text": "All", "value": "$__all"}, + "label": "Argos host" + } + ] + }, + "panels": [ + { + "id": 1, + "type": "row", + "title": "Health", + "collapsed": false, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 0} + }, + { + "id": 2, + "type": "stat", + "title": "Argos up", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 4, "w": 4, "x": 0, "y": 1}, + "targets": [ + {"refId": "A", "expr": "up{job=\"argos\", instance=~\"$instance\"}", "legendFormat": "{{instance}}"} + ], + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "background", "textMode": "value_and_name"}, + "fieldConfig": {"defaults": {"mappings": [{"type": "value", "options": {"0": {"text": "DOWN", "color": "red"}, "1": {"text": "UP", "color": "green"}}}], "thresholds": {"mode": "absolute", "steps": [{"color": "red"}, {"color": "green", "value": 1}]}}} + }, + { + "id": 3, + "type": "stat", + "title": "SearXNG instances healthy (per Argos)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 4, "w": 6, "x": 4, "y": 1}, + "targets": [ + {"refId": "A", "expr": "sum by (instance) (argos_searxng_instance_up{instance=~\"$instance\"})", "legendFormat": "{{instance}}"}, + {"refId": "B", "expr": "count by (instance) (argos_searxng_instance_up{instance=~\"$instance\"})", "legendFormat": "{{instance}} total", "hide": true} + ], + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "textMode": "value_and_name", "colorMode": "value"}, + "fieldConfig": {"defaults": {"unit": "short", "thresholds": {"mode": "absolute", "steps": [{"color": "red"}, {"color": "orange", "value": 1}, {"color": "green", "value": 2}]}}} + }, + { + "id": 4, + "type": "stat", + "title": "Tool error ratio (5m)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 4, "w": 4, "x": 10, "y": 1}, + "targets": [ + {"refId": "A", "expr": "sum(rate(argos_tool_calls_total{status=\"error\", instance=~\"$instance\"}[5m])) / clamp_min(sum(rate(argos_tool_calls_total{instance=~\"$instance\"}[5m])), 0.0001)", "legendFormat": "errors"} + ], + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "value", "textMode": "value"}, + "fieldConfig": {"defaults": {"unit": "percentunit", "thresholds": {"mode": "absolute", "steps": [{"color": "green"}, {"color": "orange", "value": 0.05}, {"color": "red", "value": 0.20}]}}} + }, + { + "id": 5, + "type": "stat", + "title": "Tool calls/sec (5m)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 4, "w": 4, "x": 14, "y": 1}, + "targets": [ + {"refId": "A", "expr": "sum(rate(argos_tool_calls_total{instance=~\"$instance\"}[5m]))", "legendFormat": "calls/s"} + ], + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "value", "textMode": "value", "graphMode": "area"}, + "fieldConfig": {"defaults": {"unit": "ops"}} + }, + { + "id": 6, + "type": "stat", + "title": "Build", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 4, "w": 6, "x": 18, "y": 1}, + "targets": [ + {"refId": "A", "expr": "argos_build_info{instance=~\"$instance\"}", "legendFormat": "{{instance}} v{{version}}"} + ], + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "textMode": "name", "colorMode": "none"}, + "fieldConfig": {"defaults": {"unit": "none"}} + }, + + { + "id": 10, + "type": "row", + "title": "Tools", + "collapsed": false, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 5} + }, + { + "id": 11, + "type": "timeseries", + "title": "Tool calls/sec by tool (5m)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 6}, + "targets": [ + {"refId": "A", "expr": "sum by (tool) (rate(argos_tool_calls_total{instance=~\"$instance\"}[5m]))", "legendFormat": "{{tool}}"} + ], + "fieldConfig": {"defaults": {"unit": "ops"}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}} + }, + { + "id": 12, + "type": "timeseries", + "title": "Tool error ratio by tool (5m)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 6}, + "targets": [ + {"refId": "A", "expr": "sum by (tool) (rate(argos_tool_calls_total{status=\"error\", instance=~\"$instance\"}[5m])) / clamp_min(sum by (tool) (rate(argos_tool_calls_total{instance=~\"$instance\"}[5m])), 0.0001)", "legendFormat": "{{tool}}"} + ], + "fieldConfig": {"defaults": {"unit": "percentunit"}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}} + }, + { + "id": 13, + "type": "timeseries", + "title": "Tool latency p50 / p95 / p99 (5m)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 14}, + "targets": [ + {"refId": "A", "expr": "histogram_quantile(0.50, sum by (le) (rate(argos_tool_duration_seconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "p50"}, + {"refId": "B", "expr": "histogram_quantile(0.95, sum by (le) (rate(argos_tool_duration_seconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "p95"}, + {"refId": "C", "expr": "histogram_quantile(0.99, sum by (le) (rate(argos_tool_duration_seconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "p99"} + ], + "fieldConfig": {"defaults": {"unit": "s"}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}} + }, + { + "id": 14, + "type": "timeseries", + "title": "Tool latency p95 by tool (5m)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 14}, + "targets": [ + {"refId": "A", "expr": "histogram_quantile(0.95, sum by (le, tool) (rate(argos_tool_duration_seconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "{{tool}}"} + ], + "fieldConfig": {"defaults": {"unit": "s"}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}} + }, + + { + "id": 20, + "type": "row", + "title": "Upstream SearXNG", + "collapsed": false, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 22} + }, + { + "id": 21, + "type": "table", + "title": "SearXNG instances (per-Argos view)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 6, "w": 12, "x": 0, "y": 23}, + "targets": [ + {"refId": "A", "expr": "argos_searxng_instance_up{instance=~\"$instance\"}", "legendFormat": "{{searxng_instance}}", "format": "table", "instant": true} + ], + "transformations": [ + {"id": "organize", "options": {"excludeByName": {"Time": true, "__name__": true, "job": true, "environment": true, "hostname": true}}} + ], + "fieldConfig": {"defaults": {"mappings": [{"type": "value", "options": {"0": {"text": "DOWN", "color": "red"}, "1": {"text": "UP", "color": "green"}}}], "custom": {"cellOptions": {"type": "color-background"}}}} + }, + { + "id": 22, + "type": "timeseries", + "title": "Upstream SearXNG requests/sec by instance (5m)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 6, "w": 12, "x": 12, "y": 23}, + "targets": [ + {"refId": "A", "expr": "sum by (instance, searxng_instance) (rate(argos_searxng_requests_total{instance=~\"$instance\"}[5m]))", "legendFormat": "{{instance}} → {{searxng_instance}}"} + ], + "fieldConfig": {"defaults": {"unit": "ops"}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}} + }, + { + "id": 23, + "type": "timeseries", + "title": "Upstream SearXNG error ratio by instance (5m)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 6, "w": 12, "x": 0, "y": 29}, + "targets": [ + {"refId": "A", "expr": "sum by (searxng_instance) (rate(argos_searxng_requests_total{status=\"error\", instance=~\"$instance\"}[5m])) / clamp_min(sum by (searxng_instance) (rate(argos_searxng_requests_total{instance=~\"$instance\"}[5m])), 0.0001)", "legendFormat": "{{searxng_instance}}"} + ], + "fieldConfig": {"defaults": {"unit": "percentunit"}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}} + }, + { + "id": 24, + "type": "timeseries", + "title": "Upstream SearXNG latency p95 by instance (5m)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 6, "w": 12, "x": 12, "y": 29}, + "targets": [ + {"refId": "A", "expr": "histogram_quantile(0.95, sum by (le, searxng_instance) (rate(argos_searxng_request_duration_seconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "{{searxng_instance}} p95"} + ], + "fieldConfig": {"defaults": {"unit": "s"}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}} + }, + + { + "id": 30, + "type": "row", + "title": "Cache & webpage fetch", + "collapsed": false, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 35} + }, + { + "id": 31, + "type": "stat", + "title": "Cache hit ratio (5m)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 4, "w": 6, "x": 0, "y": 36}, + "targets": [ + {"refId": "A", "expr": "sum(rate(argos_cache_operations_total{operation=\"get\", result=\"hit\", instance=~\"$instance\"}[5m])) / clamp_min(sum(rate(argos_cache_operations_total{operation=\"get\", instance=~\"$instance\"}[5m])), 0.0001)", "legendFormat": "hits"} + ], + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "value", "textMode": "value", "graphMode": "area"}, + "fieldConfig": {"defaults": {"unit": "percentunit", "thresholds": {"mode": "absolute", "steps": [{"color": "red"}, {"color": "orange", "value": 0.10}, {"color": "green", "value": 0.30}]}}} + }, + { + "id": 32, + "type": "timeseries", + "title": "Cache ops/sec by result (5m)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 8, "w": 9, "x": 6, "y": 36}, + "targets": [ + {"refId": "A", "expr": "sum by (operation, result) (rate(argos_cache_operations_total{instance=~\"$instance\"}[5m]))", "legendFormat": "{{operation}}/{{result}}"} + ], + "fieldConfig": {"defaults": {"unit": "ops"}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}} + }, + { + "id": 33, + "type": "timeseries", + "title": "Webpage fetch outcomes/sec (5m)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 8, "w": 9, "x": 15, "y": 36}, + "targets": [ + {"refId": "A", "expr": "sum by (status) (rate(argos_webpage_fetch_total{instance=~\"$instance\"}[5m]))", "legendFormat": "{{status}}"} + ], + "fieldConfig": {"defaults": {"unit": "ops"}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}} + }, + + { + "id": 90, + "type": "row", + "title": "Logs", + "collapsed": false, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 44} + }, + { + "id": 91, + "type": "logs", + "title": "argos (Loki)", + "datasource": {"type": "loki", "uid": "${loki}"}, + "gridPos": {"h": 12, "w": 24, "x": 0, "y": 45}, + "targets": [ + {"refId": "A", "expr": "{job=\"argos\"}"} + ], + "options": {"showTime": true, "wrapLogMessage": true, "enableLogDetails": true, "dedupStrategy": "none"} + } + ] +} diff --git a/dashboards/daedalus_stack.json b/dashboards/daedalus_stack.json new file mode 100644 index 0000000..1e93306 --- /dev/null +++ b/dashboards/daedalus_stack.json @@ -0,0 +1,702 @@ +{ + "title": "Daedalus Stack", + "uid": "daedalus-stack", + "tags": ["daedalus", "mnemosyne", "pallas", "ouranos"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": true, + "fiscalYearStartMonth": 0, + "weekStart": "", + "refresh": "30s", + "time": {"from": "now-1h", "to": "now"}, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "Neo4j dashboard", + "tooltip": "Detailed Neo4j metrics (ariel, umbriel)", + "type": "link", + "url": "/d/neo4j" + }, + { + "asDropdown": false, + "icon": "doc", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "Explore Logs", + "tooltip": "Loki: daedalus + mnemosyne + pallas", + "type": "link", + "url": "/explore?orgId=1&left=%7B%22datasource%22:%22Loki%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bservice%3D~%5C%22daedalus%7Cmnemosyne%7Cpallas%5C%22%7D%22%7D%5D%7D" + } + ], + "templating": { + "list": [ + { + "name": "prom", + "type": "datasource", + "query": "prometheus", + "current": {"selected": false, "text": "Prometheus", "value": "Prometheus"}, + "hide": 0, + "label": "Prometheus datasource" + }, + { + "name": "loki", + "type": "datasource", + "query": "loki", + "current": {"selected": false, "text": "Loki", "value": "Loki"}, + "hide": 0, + "label": "Loki datasource" + }, + { + "name": "pallas_inst", + "type": "query", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "query": "label_values(up{job=\"pallas\"}, instance)", + "refresh": 1, + "includeAll": true, + "multi": true, + "current": {"selected": true, "text": "All", "value": "$__all"}, + "label": "Pallas instance" + }, + { + "name": "agent", + "type": "query", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "query": "label_values(pallas_send_message_total{instance=~\"$pallas_inst\"}, agent)", + "refresh": 2, + "includeAll": true, + "multi": true, + "current": {"selected": true, "text": "All", "value": "$__all"}, + "label": "Agent" + } + ] + }, + "panels": [ + { + "id": 100, + "type": "row", + "title": "Summary", + "collapsed": false, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 0} + }, + { + "id": 101, + "type": "stat", + "title": "Daedalus", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 4, "w": 3, "x": 0, "y": 1}, + "targets": [ + {"refId": "A", "expr": "up{job=\"daedalus\"}", "legendFormat": "{{instance}}"} + ], + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "background", "textMode": "value_and_name"}, + "fieldConfig": {"defaults": {"mappings": [{"type": "value", "options": {"0": {"text": "DOWN", "color": "red"}, "1": {"text": "UP", "color": "green"}}}], "thresholds": {"mode": "absolute", "steps": [{"color": "red"}, {"color": "green", "value": 1}]}}} + }, + { + "id": 102, + "type": "stat", + "title": "Mnemosyne app", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 4, "w": 3, "x": 3, "y": 1}, + "targets": [ + {"refId": "A", "expr": "up{job=\"mnemosyne\", component=\"app\"}", "legendFormat": "app"} + ], + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "background", "textMode": "value_and_name"}, + "fieldConfig": {"defaults": {"mappings": [{"type": "value", "options": {"0": {"text": "DOWN", "color": "red"}, "1": {"text": "UP", "color": "green"}}}], "thresholds": {"mode": "absolute", "steps": [{"color": "red"}, {"color": "green", "value": 1}]}}} + }, + { + "id": 103, + "type": "stat", + "title": "Mnemosyne web", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 4, "w": 3, "x": 6, "y": 1}, + "targets": [ + {"refId": "A", "expr": "up{job=\"mnemosyne\", component=\"web\"}", "legendFormat": "web"} + ], + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "background", "textMode": "value_and_name"}, + "fieldConfig": {"defaults": {"mappings": [{"type": "value", "options": {"0": {"text": "DOWN", "color": "red"}, "1": {"text": "UP", "color": "green"}}}], "thresholds": {"mode": "absolute", "steps": [{"color": "red"}, {"color": "green", "value": 1}]}}} + }, + { + "id": 104, + "type": "stat", + "title": "Pallas up ratio", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 4, "w": 3, "x": 9, "y": 1}, + "targets": [ + {"refId": "A", "expr": "sum(up{job=\"pallas\"}) / count(up{job=\"pallas\"})", "legendFormat": "up ratio"} + ], + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "value", "textMode": "value"}, + "fieldConfig": {"defaults": {"unit": "percentunit", "thresholds": {"mode": "absolute", "steps": [{"color": "red"}, {"color": "orange", "value": 0.67}, {"color": "green", "value": 1}]}}} + }, + { + "id": 105, + "type": "stat", + "title": "Agents healthy", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 4, "w": 3, "x": 12, "y": 1}, + "targets": [ + {"refId": "A", "expr": "sum(daedalus_agents_by_health{status=\"ok\"}) / clamp_min(daedalus_agents_total, 1)", "legendFormat": "healthy"} + ], + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "value", "textMode": "value"}, + "fieldConfig": {"defaults": {"unit": "percentunit", "thresholds": {"mode": "absolute", "steps": [{"color": "red"}, {"color": "orange", "value": 0.7}, {"color": "green", "value": 1}]}}} + }, + { + "id": 106, + "type": "stat", + "title": "Chat p95 (5m)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 4, "w": 3, "x": 15, "y": 1}, + "targets": [ + {"refId": "A", "expr": "histogram_quantile(0.95, sum by (le) (rate(daedalus_agent_response_duration_seconds_bucket{source=\"chat\"}[5m])))", "legendFormat": "chat p95"} + ], + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "value", "textMode": "value"}, + "fieldConfig": {"defaults": {"unit": "s", "thresholds": {"mode": "absolute", "steps": [{"color": "green"}, {"color": "orange", "value": 10}, {"color": "red", "value": 30}]}}} + }, + { + "id": 107, + "type": "timeseries", + "title": "Stack up (last hour)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 4, "w": 6, "x": 18, "y": 1}, + "targets": [ + {"refId": "A", "expr": "up{job=~\"daedalus|mnemosyne|pallas\"}", "legendFormat": "{{job}} {{instance}} {{component}}"} + ], + "fieldConfig": {"defaults": {"unit": "short", "min": 0, "max": 1, "custom": {"drawStyle": "line", "lineInterpolation": "stepBefore", "fillOpacity": 10}}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}} + }, + + { + "id": 200, + "type": "row", + "title": "Daedalus", + "collapsed": false, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 5} + }, + { + "id": 201, + "type": "stat", + "title": "Daedalus up", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 4, "w": 4, "x": 0, "y": 6}, + "targets": [ + {"refId": "A", "expr": "daedalus_up", "legendFormat": "daedalus"} + ], + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "background", "textMode": "value_and_name"}, + "fieldConfig": {"defaults": {"mappings": [{"type": "value", "options": {"0": {"text": "DOWN", "color": "red"}, "1": {"text": "UP", "color": "green"}}}], "thresholds": {"mode": "absolute", "steps": [{"color": "red"}, {"color": "green", "value": 1}]}}} + }, + { + "id": 202, + "type": "stat", + "title": "5xx error rate (5m)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 4, "w": 4, "x": 4, "y": 6}, + "targets": [ + {"refId": "A", "expr": "sum(rate(daedalus_http_requests_total{status=~\"5..\"}[5m])) / clamp_min(sum(rate(daedalus_http_requests_total[5m])), 0.0001)", "legendFormat": "5xx"} + ], + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "value", "textMode": "value"}, + "fieldConfig": {"defaults": {"unit": "percentunit", "thresholds": {"mode": "absolute", "steps": [{"color": "green"}, {"color": "orange", "value": 0.01}, {"color": "red", "value": 0.05}]}}} + }, + { + "id": 203, + "type": "stat", + "title": "MCP connections active", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 4, "w": 4, "x": 8, "y": 6}, + "targets": [ + {"refId": "A", "expr": "sum(daedalus_mcp_connections_active)", "legendFormat": "active"} + ], + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "value", "textMode": "value"}, + "fieldConfig": {"defaults": {"unit": "short", "thresholds": {"mode": "absolute", "steps": [{"color": "red"}, {"color": "green", "value": 1}]}}} + }, + { + "id": 204, + "type": "stat", + "title": "Avg context window %", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 4, "w": 4, "x": 12, "y": 6}, + "targets": [ + {"refId": "A", "expr": "avg(daedalus_chat_context_pct)", "legendFormat": "avg"} + ], + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "value", "textMode": "value"}, + "fieldConfig": {"defaults": {"unit": "percent", "thresholds": {"mode": "absolute", "steps": [{"color": "green"}, {"color": "orange", "value": 70}, {"color": "red", "value": 90}]}}} + }, + { + "id": 205, + "type": "stat", + "title": "Tokens/sec (5m, total)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 4, "w": 8, "x": 16, "y": 6}, + "targets": [ + {"refId": "A", "expr": "sum(rate(daedalus_chat_tokens_total[5m]))", "legendFormat": "tok/s"} + ], + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "value", "textMode": "value", "graphMode": "area"}, + "fieldConfig": {"defaults": {"unit": "short"}} + }, + { + "id": 210, + "type": "timeseries", + "title": "Chat latency (p50/p95/p99)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 10}, + "targets": [ + {"refId": "A", "expr": "histogram_quantile(0.50, sum by (le) (rate(daedalus_agent_response_duration_seconds_bucket{source=\"chat\"}[5m])))", "legendFormat": "p50"}, + {"refId": "B", "expr": "histogram_quantile(0.95, sum by (le) (rate(daedalus_agent_response_duration_seconds_bucket{source=\"chat\"}[5m])))", "legendFormat": "p95"}, + {"refId": "C", "expr": "histogram_quantile(0.99, sum by (le) (rate(daedalus_agent_response_duration_seconds_bucket{source=\"chat\"}[5m])))", "legendFormat": "p99"} + ], + "fieldConfig": {"defaults": {"unit": "s"}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}} + }, + { + "id": 211, + "type": "timeseries", + "title": "Voice pipeline p95 (STT / agent / TTS / total)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 10}, + "targets": [ + {"refId": "A", "expr": "histogram_quantile(0.95, sum by (le) (rate(daedalus_voice_stt_duration_seconds_bucket[5m])))", "legendFormat": "stt"}, + {"refId": "B", "expr": "histogram_quantile(0.95, sum by (le) (rate(daedalus_voice_agent_duration_seconds_bucket[5m])))", "legendFormat": "agent"}, + {"refId": "C", "expr": "histogram_quantile(0.95, sum by (le) (rate(daedalus_voice_tts_duration_seconds_bucket[5m])))", "legendFormat": "tts"}, + {"refId": "D", "expr": "histogram_quantile(0.95, sum by (le) (rate(daedalus_voice_pipeline_duration_seconds_bucket[5m])))", "legendFormat": "total"} + ], + "fieldConfig": {"defaults": {"unit": "s", "custom": {"stacking": {"mode": "none"}}}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}} + }, + { + "id": 220, + "type": "timeseries", + "title": "Pallas reach — MCP error ratio by server (5m)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 18}, + "targets": [ + {"refId": "A", "expr": "sum by (server) (rate(daedalus_mcp_requests_total{status=\"error\"}[5m])) / clamp_min(sum by (server) (rate(daedalus_mcp_requests_total[5m])), 0.0001)", "legendFormat": "{{server}}"} + ], + "fieldConfig": {"defaults": {"unit": "percentunit"}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}} + }, + { + "id": 221, + "type": "timeseries", + "title": "Mnemosyne reach — p95 latency by operation (5m)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 18}, + "targets": [ + {"refId": "A", "expr": "histogram_quantile(0.95, sum by (le, operation) (rate(daedalus_mnemosyne_request_duration_seconds_bucket[5m])))", "legendFormat": "{{operation}} p95"}, + {"refId": "B", "expr": "sum(rate(daedalus_mnemosyne_requests_total{status=\"error\"}[5m]))", "legendFormat": "errors/s (right)"} + ], + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": [{"matcher": {"id": "byName", "options": "errors/s (right)"}, "properties": [{"id": "unit", "value": "ops"}, {"id": "custom.axisPlacement", "value": "right"}]}]}, + "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}} + }, + { + "id": 230, + "type": "timeseries", + "title": "Token burn by direction (tokens/sec, 5m)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 26}, + "targets": [ + {"refId": "A", "expr": "sum by (direction) (rate(daedalus_chat_tokens_total[5m]))", "legendFormat": "{{direction}}"} + ], + "fieldConfig": {"defaults": {"unit": "short", "custom": {"drawStyle": "line", "fillOpacity": 20, "stacking": {"mode": "normal"}}}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}} + }, + { + "id": 231, + "type": "timeseries", + "title": "Mnemosyne ingest jobs (status, 5m)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 26}, + "targets": [ + {"refId": "A", "expr": "sum by (status) (rate(daedalus_mnemosyne_ingest_jobs_total[5m]))", "legendFormat": "{{status}}"} + ], + "fieldConfig": {"defaults": {"unit": "ops"}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}} + }, + + { + "id": 300, + "type": "row", + "title": "Mnemosyne", + "collapsed": false, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 34} + }, + { + "id": 301, + "type": "stat", + "title": "App", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 4, "w": 4, "x": 0, "y": 35}, + "targets": [ + {"refId": "A", "expr": "up{job=\"mnemosyne\", component=\"app\"}", "legendFormat": "app"} + ], + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "background", "textMode": "value_and_name"}, + "fieldConfig": {"defaults": {"mappings": [{"type": "value", "options": {"0": {"text": "DOWN", "color": "red"}, "1": {"text": "UP", "color": "green"}}}], "thresholds": {"mode": "absolute", "steps": [{"color": "red"}, {"color": "green", "value": 1}]}}} + }, + { + "id": 302, + "type": "stat", + "title": "Web (nginx)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 4, "w": 4, "x": 4, "y": 35}, + "targets": [ + {"refId": "A", "expr": "up{job=\"mnemosyne\", component=\"web\"}", "legendFormat": "web"} + ], + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "background", "textMode": "value_and_name"}, + "fieldConfig": {"defaults": {"mappings": [{"type": "value", "options": {"0": {"text": "DOWN", "color": "red"}, "1": {"text": "UP", "color": "green"}}}], "thresholds": {"mode": "absolute", "steps": [{"color": "red"}, {"color": "green", "value": 1}]}}} + }, + { + "id": 303, + "type": "stat", + "title": "Search rate (5m)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 4, "w": 4, "x": 8, "y": 35}, + "targets": [ + {"refId": "A", "expr": "sum(rate(mnemosyne_search_requests_total[5m]))", "legendFormat": "req/s"} + ], + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "value", "textMode": "value", "graphMode": "area"}, + "fieldConfig": {"defaults": {"unit": "reqps"}} + }, + { + "id": 304, + "type": "stat", + "title": "Embedding queue depth", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 4, "w": 4, "x": 12, "y": 35}, + "targets": [ + {"refId": "A", "expr": "mnemosyne_embedding_queue_size", "legendFormat": "queue"} + ], + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "value", "textMode": "value", "graphMode": "area"}, + "fieldConfig": {"defaults": {"unit": "short", "thresholds": {"mode": "absolute", "steps": [{"color": "green"}, {"color": "orange", "value": 10}, {"color": "red", "value": 100}]}}} + }, + { + "id": 305, + "type": "stat", + "title": "Pipeline in-progress", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 4, "w": 4, "x": 16, "y": 35}, + "targets": [ + {"refId": "A", "expr": "mnemosyne_pipeline_items_in_progress", "legendFormat": "in-flight"} + ], + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "value", "textMode": "value", "graphMode": "area"}, + "fieldConfig": {"defaults": {"unit": "short"}} + }, + { + "id": 306, + "type": "stat", + "title": "MCP tool error rate (5m)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 4, "w": 4, "x": 20, "y": 35}, + "targets": [ + {"refId": "A", "expr": "sum(rate(mcp_tool_invocations_total{status=\"error\"}[5m])) / clamp_min(sum(rate(mcp_tool_invocations_total[5m])), 0.0001)", "legendFormat": "err"} + ], + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "value", "textMode": "value"}, + "fieldConfig": {"defaults": {"unit": "percentunit", "thresholds": {"mode": "absolute", "steps": [{"color": "green"}, {"color": "orange", "value": 0.01}, {"color": "red", "value": 0.05}]}}} + }, + { + "id": 310, + "type": "timeseries", + "title": "Search rate by type (5m)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 39}, + "targets": [ + {"refId": "A", "expr": "sum by (search_type) (rate(mnemosyne_search_requests_total[5m]))", "legendFormat": "{{search_type}}"} + ], + "fieldConfig": {"defaults": {"unit": "reqps"}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}} + }, + { + "id": 311, + "type": "timeseries", + "title": "Search latency p95 by type (5m)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 39}, + "targets": [ + {"refId": "A", "expr": "histogram_quantile(0.95, sum by (le, search_type) (rate(mnemosyne_search_duration_seconds_bucket[5m])))", "legendFormat": "{{search_type}} p95"}, + {"refId": "B", "expr": "histogram_quantile(0.95, sum by (le) (rate(mnemosyne_search_total_duration_seconds_bucket[5m])))", "legendFormat": "end-to-end p95"} + ], + "fieldConfig": {"defaults": {"unit": "s"}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}} + }, + { + "id": 320, + "type": "timeseries", + "title": "Embedding queue depth over time", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 8, "w": 8, "x": 0, "y": 47}, + "targets": [ + {"refId": "A", "expr": "mnemosyne_embedding_queue_size", "legendFormat": "queue"} + ], + "fieldConfig": {"defaults": {"unit": "short", "custom": {"drawStyle": "line", "fillOpacity": 20}}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}} + }, + { + "id": 321, + "type": "timeseries", + "title": "Embeddings generated (per sec, by model)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 8, "w": 8, "x": 8, "y": 47}, + "targets": [ + {"refId": "A", "expr": "sum by (model_name) (rate(mnemosyne_embeddings_generated_total[5m]))", "legendFormat": "{{model_name}}"} + ], + "fieldConfig": {"defaults": {"unit": "ops"}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}} + }, + { + "id": 322, + "type": "timeseries", + "title": "Pipeline items (per sec, by status)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 47}, + "targets": [ + {"refId": "A", "expr": "sum by (status) (rate(mnemosyne_pipeline_items_total[5m]))", "legendFormat": "{{status}}"}, + {"refId": "B", "expr": "sum by (error_type) (rate(mnemosyne_embedding_api_errors_total[5m]))", "legendFormat": "api err: {{error_type}}"} + ], + "fieldConfig": {"defaults": {"unit": "ops"}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}} + }, + { + "id": 330, + "type": "timeseries", + "title": "Neo4j @ umbriel — transactions (rate / open)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 55}, + "targets": [ + {"refId": "A", "expr": "rate(neo4j_monitor_tx_totalOpenedTx{instance=~\"umbriel.*\"}[5m])", "legendFormat": "{{instance}} open rate"}, + {"refId": "B", "expr": "neo4j_monitor_tx_currentOpenedTx{instance=~\"umbriel.*\"}", "legendFormat": "{{instance}} current open"} + ], + "fieldConfig": {"defaults": {"unit": "short"}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}} + }, + { + "id": 331, + "type": "stat", + "title": "Neo4j @ umbriel — rollback ratio (10m)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 4, "w": 6, "x": 12, "y": 55}, + "targets": [ + {"refId": "A", "expr": "rate(neo4j_monitor_tx_rolledBackTx{instance=~\"umbriel.*\"}[10m]) / clamp_min(rate(neo4j_monitor_tx_totalOpenedTx{instance=~\"umbriel.*\"}[10m]), 0.0001)", "legendFormat": "{{instance}}"} + ], + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "textMode": "value_and_name"}, + "fieldConfig": {"defaults": {"unit": "percentunit", "thresholds": {"mode": "absolute", "steps": [{"color": "green"}, {"color": "orange", "value": 0.05}, {"color": "red", "value": 0.10}]}}} + }, + { + "id": 332, + "type": "stat", + "title": "Neo4j @ umbriel — store size", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 4, "w": 6, "x": 18, "y": 55}, + "targets": [ + {"refId": "A", "expr": "neo4j_monitor_store_totalStoreSize{instance=~\"umbriel.*\"}", "legendFormat": "{{instance}}"} + ], + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "textMode": "value_and_name"}, + "fieldConfig": {"defaults": {"unit": "bytes"}} + }, + + { + "id": 400, + "type": "row", + "title": "Pallas Agents", + "collapsed": false, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 63} + }, + { + "id": 401, + "type": "stat", + "title": "Instance up", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 4, "w": 6, "x": 0, "y": 64}, + "targets": [ + {"refId": "A", "expr": "up{job=\"pallas\", instance=~\"$pallas_inst\"}", "legendFormat": "{{instance}}"} + ], + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": ""}, "colorMode": "background", "textMode": "value_and_name"}, + "fieldConfig": {"defaults": {"mappings": [{"type": "value", "options": {"0": {"text": "DOWN", "color": "red"}, "1": {"text": "UP", "color": "green"}}}], "thresholds": {"mode": "absolute", "steps": [{"color": "red"}, {"color": "green", "value": 1}]}}} + }, + { + "id": 402, + "type": "stat", + "title": "Aggregate agent health (min)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 4, "w": 6, "x": 6, "y": 64}, + "targets": [ + {"refId": "A", "expr": "min by (instance) (pallas_agent_health_status{instance=~\"$pallas_inst\"})", "legendFormat": "{{instance}}"} + ], + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "value", "textMode": "value_and_name"}, + "fieldConfig": {"defaults": {"mappings": [{"type": "value", "options": {"0": {"text": "ERROR", "color": "red"}, "0.5": {"text": "DEGRADED", "color": "orange"}, "1": {"text": "OK", "color": "green"}}}], "thresholds": {"mode": "absolute", "steps": [{"color": "red"}, {"color": "orange", "value": 0.5}, {"color": "green", "value": 1}]}}} + }, + { + "id": 403, + "type": "stat", + "title": "Downstream MCPs up (ratio)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 4, "w": 6, "x": 12, "y": 64}, + "targets": [ + {"refId": "A", "expr": "sum by (instance) (pallas_downstream_up{instance=~\"$pallas_inst\"}) / clamp_min(count by (instance) (pallas_downstream_up{instance=~\"$pallas_inst\"}), 1)", "legendFormat": "{{instance}}"} + ], + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "value", "textMode": "value_and_name"}, + "fieldConfig": {"defaults": {"unit": "percentunit", "thresholds": {"mode": "absolute", "steps": [{"color": "red"}, {"color": "orange", "value": 0.5}, {"color": "green", "value": 1}]}}} + }, + { + "id": 404, + "type": "stat", + "title": "Turn error ratio (5m)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 4, "w": 6, "x": 18, "y": 64}, + "targets": [ + {"refId": "A", "expr": "sum by (instance) (rate(pallas_send_message_total{outcome=\"error\", instance=~\"$pallas_inst\"}[5m])) / clamp_min(sum by (instance) (rate(pallas_send_message_total{instance=~\"$pallas_inst\"}[5m])), 0.0001)", "legendFormat": "{{instance}}"} + ], + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "value", "textMode": "value_and_name"}, + "fieldConfig": {"defaults": {"unit": "percentunit", "thresholds": {"mode": "absolute", "steps": [{"color": "green"}, {"color": "orange", "value": 0.01}, {"color": "red", "value": 0.05}]}}} + }, + { + "id": 410, + "type": "timeseries", + "title": "Turn latency p95 by agent (5m)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 68}, + "targets": [ + {"refId": "A", "expr": "histogram_quantile(0.95, sum by (le, agent, instance) (rate(pallas_send_message_duration_seconds_bucket{instance=~\"$pallas_inst\", agent=~\"$agent\"}[5m])))", "legendFormat": "{{instance}}/{{agent}}"} + ], + "fieldConfig": {"defaults": {"unit": "s"}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}} + }, + { + "id": 411, + "type": "table", + "title": "Long-running agents — p99 turn (5m)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 68}, + "targets": [ + {"refId": "A", "expr": "histogram_quantile(0.99, sum by (agent, instance, le) (rate(pallas_send_message_duration_seconds_bucket{instance=~\"$pallas_inst\", agent=~\"$agent\"}[5m])))", "legendFormat": "", "format": "table", "instant": true} + ], + "fieldConfig": {"defaults": {"unit": "s", "custom": {"align": "auto"}, "thresholds": {"mode": "absolute", "steps": [{"color": "green"}, {"color": "orange", "value": 30}, {"color": "red", "value": 60}]}, "color": {"mode": "thresholds"}}, "overrides": [{"matcher": {"id": "byName", "options": "Value"}, "properties": [{"id": "displayName", "value": "p99 (s)"}, {"id": "custom.cellOptions", "value": {"type": "color-background"}}]}]}, + "options": {"showHeader": true, "sortBy": [{"displayName": "p99 (s)", "desc": true}]}, + "transformations": [{"id": "organize", "options": {"excludeByName": {"Time": true, "__name__": true, "job": true}}}] + }, + { + "id": 420, + "type": "timeseries", + "title": "Turn errors per agent (15m increase)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 76}, + "targets": [ + {"refId": "A", "expr": "sum by (agent, instance) (increase(pallas_send_message_total{outcome=\"error\", instance=~\"$pallas_inst\", agent=~\"$agent\"}[15m]))", "legendFormat": "{{instance}}/{{agent}}"} + ], + "fieldConfig": {"defaults": {"unit": "short"}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}} + }, + { + "id": 421, + "type": "timeseries", + "title": "Tokens/sec by kind (5m)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 76}, + "targets": [ + {"refId": "A", "expr": "sum by (kind) (rate(pallas_llm_tokens_total{instance=~\"$pallas_inst\", agent=~\"$agent\"}[5m]))", "legendFormat": "{{kind}}"} + ], + "fieldConfig": {"defaults": {"unit": "short", "custom": {"drawStyle": "line", "fillOpacity": 20, "stacking": {"mode": "normal"}}}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}} + }, + { + "id": 422, + "type": "table", + "title": "Top-burning agents (24h, input+output tokens)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 84}, + "targets": [ + {"refId": "A", "expr": "topk(10, sum by (agent, model, instance) (increase(pallas_llm_tokens_total{kind=~\"input|output\", instance=~\"$pallas_inst\", agent=~\"$agent\"}[24h])))", "legendFormat": "", "format": "table", "instant": true} + ], + "fieldConfig": {"defaults": {"unit": "short"}, "overrides": [{"matcher": {"id": "byName", "options": "Value"}, "properties": [{"id": "displayName", "value": "tokens (24h)"}, {"id": "custom.cellOptions", "value": {"type": "gauge", "mode": "gradient"}}]}]}, + "options": {"showHeader": true, "sortBy": [{"displayName": "tokens (24h)", "desc": true}]}, + "transformations": [{"id": "organize", "options": {"excludeByName": {"Time": true, "__name__": true, "job": true}}}] + }, + { + "id": 423, + "type": "stat", + "title": "Cache effectiveness (1h)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 4, "w": 6, "x": 12, "y": 84}, + "targets": [ + {"refId": "A", "expr": "sum(rate(pallas_llm_tokens_total{kind=\"cache_read\", instance=~\"$pallas_inst\"}[1h])) / clamp_min(sum(rate(pallas_llm_tokens_total{kind=~\"input|cache_read\", instance=~\"$pallas_inst\"}[1h])), 0.0001)", "legendFormat": "cache hit"} + ], + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "value", "textMode": "value", "graphMode": "area"}, + "fieldConfig": {"defaults": {"unit": "percentunit", "thresholds": {"mode": "absolute", "steps": [{"color": "red"}, {"color": "orange", "value": 0.2}, {"color": "green", "value": 0.5}]}}} + }, + { + "id": 424, + "type": "stat", + "title": "LLM turns/sec (5m)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 4, "w": 6, "x": 18, "y": 84}, + "targets": [ + {"refId": "A", "expr": "sum(rate(pallas_llm_turns_total{instance=~\"$pallas_inst\", agent=~\"$agent\"}[5m]))", "legendFormat": "turns/s"} + ], + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "value", "textMode": "value", "graphMode": "area"}, + "fieldConfig": {"defaults": {"unit": "ops"}} + }, + { + "id": 430, + "type": "timeseries", + "title": "Cypher tool calls — rate by outcome (Pallas → ariel)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 92}, + "targets": [ + {"refId": "A", "expr": "sum by (outcome) (rate(pallas_tool_calls_total{server=\"neo4j_cypher\", instance=~\"$pallas_inst\", agent=~\"$agent\"}[5m]))", "legendFormat": "{{outcome}}"} + ], + "fieldConfig": {"defaults": {"unit": "ops"}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}} + }, + { + "id": 431, + "type": "timeseries", + "title": "Cypher tool calls — p95 latency by agent", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 92}, + "targets": [ + {"refId": "A", "expr": "histogram_quantile(0.95, sum by (le, agent, instance) (rate(pallas_tool_call_duration_seconds_bucket{server=\"neo4j_cypher\", instance=~\"$pallas_inst\", agent=~\"$agent\"}[5m])))", "legendFormat": "{{instance}}/{{agent}}"} + ], + "fieldConfig": {"defaults": {"unit": "s"}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}} + }, + { + "id": 440, + "type": "timeseries", + "title": "Neo4j @ ariel — transactions (rate / open)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 100}, + "targets": [ + {"refId": "A", "expr": "rate(neo4j_monitor_tx_totalOpenedTx{instance=~\"ariel.*\"}[5m])", "legendFormat": "{{instance}} open rate"}, + {"refId": "B", "expr": "neo4j_monitor_tx_currentOpenedTx{instance=~\"ariel.*\"}", "legendFormat": "{{instance}} current open"} + ], + "fieldConfig": {"defaults": {"unit": "short"}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}} + }, + { + "id": 441, + "type": "stat", + "title": "Neo4j @ ariel — rollback ratio (10m)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 4, "w": 6, "x": 12, "y": 100}, + "targets": [ + {"refId": "A", "expr": "rate(neo4j_monitor_tx_rolledBackTx{instance=~\"ariel.*\"}[10m]) / clamp_min(rate(neo4j_monitor_tx_totalOpenedTx{instance=~\"ariel.*\"}[10m]), 0.0001)", "legendFormat": "{{instance}}"} + ], + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "textMode": "value_and_name"}, + "fieldConfig": {"defaults": {"unit": "percentunit", "thresholds": {"mode": "absolute", "steps": [{"color": "green"}, {"color": "orange", "value": 0.05}, {"color": "red", "value": 0.10}]}}} + }, + { + "id": 442, + "type": "stat", + "title": "Neo4j @ ariel — store size", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 4, "w": 6, "x": 18, "y": 100}, + "targets": [ + {"refId": "A", "expr": "neo4j_monitor_store_totalStoreSize{instance=~\"ariel.*\"}", "legendFormat": "{{instance}}"} + ], + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "textMode": "value_and_name"}, + "fieldConfig": {"defaults": {"unit": "bytes"}} + } + ] +} diff --git a/dashboards/searxng.json b/dashboards/searxng.json new file mode 100644 index 0000000..5fb9be1 --- /dev/null +++ b/dashboards/searxng.json @@ -0,0 +1,202 @@ +{ + "title": "SearXNG", + "uid": "searxng", + "tags": ["searxng", "argos", "ouranos"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": true, + "fiscalYearStartMonth": 0, + "weekStart": "", + "refresh": "30s", + "time": {"from": "now-1h", "to": "now"}, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "Argos dashboard", + "tooltip": "Argos MCP server using these SearXNG instances", + "type": "link", + "url": "/d/argos" + }, + { + "asDropdown": false, + "icon": "doc", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "SearXNG logs", + "tooltip": "Loki: {job=\"searxng\"}", + "type": "link", + "url": "/explore?orgId=1&left=%7B%22datasource%22:%22Loki%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bjob%3D%5C%22searxng%5C%22%7D%22%7D%5D%7D" + } + ], + "templating": { + "list": [ + { + "name": "prom", + "type": "datasource", + "query": "prometheus", + "current": {"selected": false, "text": "Prometheus", "value": "Prometheus"}, + "hide": 0, + "label": "Prometheus datasource" + }, + { + "name": "loki", + "type": "datasource", + "query": "loki", + "current": {"selected": false, "text": "Loki", "value": "Loki"}, + "hide": 0, + "label": "Loki datasource" + }, + { + "name": "host", + "type": "query", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "query": "label_values(probe_success{service=\"searxng\"}, hostname)", + "refresh": 1, + "includeAll": true, + "multi": true, + "current": {"selected": true, "text": "All", "value": "$__all"}, + "label": "SearXNG host" + } + ] + }, + "panels": [ + { + "id": 1, + "type": "row", + "title": "Independent probe (Alloy blackbox /healthz)", + "collapsed": false, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 0} + }, + { + "id": 2, + "type": "stat", + "title": "SearXNG /healthz", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 5, "w": 8, "x": 0, "y": 1}, + "targets": [ + {"refId": "A", "expr": "probe_success{service=\"searxng\", hostname=~\"$host\"}", "legendFormat": "{{hostname}}"} + ], + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "background", "textMode": "value_and_name"}, + "fieldConfig": {"defaults": {"mappings": [{"type": "value", "options": {"0": {"text": "DOWN", "color": "red"}, "1": {"text": "UP", "color": "green"}}}], "thresholds": {"mode": "absolute", "steps": [{"color": "red"}, {"color": "green", "value": 1}]}}} + }, + { + "id": 3, + "type": "stat", + "title": "Last probe HTTP status", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 5, "w": 8, "x": 8, "y": 1}, + "targets": [ + {"refId": "A", "expr": "probe_http_status_code{service=\"searxng\", hostname=~\"$host\"}", "legendFormat": "{{hostname}}"} + ], + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "value", "textMode": "value_and_name"}, + "fieldConfig": {"defaults": {"unit": "short", "thresholds": {"mode": "absolute", "steps": [{"color": "red"}, {"color": "green", "value": 200}, {"color": "orange", "value": 300}, {"color": "red", "value": 400}]}}} + }, + { + "id": 4, + "type": "stat", + "title": "Probe duration (last)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 5, "w": 8, "x": 16, "y": 1}, + "targets": [ + {"refId": "A", "expr": "probe_duration_seconds{service=\"searxng\", hostname=~\"$host\"}", "legendFormat": "{{hostname}}"} + ], + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "value", "textMode": "value_and_name", "graphMode": "area"}, + "fieldConfig": {"defaults": {"unit": "s", "thresholds": {"mode": "absolute", "steps": [{"color": "green"}, {"color": "orange", "value": 1}, {"color": "red", "value": 3}]}}} + }, + { + "id": 5, + "type": "timeseries", + "title": "Probe success over time", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 6}, + "targets": [ + {"refId": "A", "expr": "probe_success{service=\"searxng\", hostname=~\"$host\"}", "legendFormat": "{{hostname}}"} + ], + "fieldConfig": {"defaults": {"unit": "short", "min": 0, "max": 1, "custom": {"drawStyle": "line", "lineWidth": 2, "fillOpacity": 20}}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}} + }, + { + "id": 6, + "type": "timeseries", + "title": "Probe duration over time", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 6}, + "targets": [ + {"refId": "A", "expr": "probe_duration_seconds{service=\"searxng\", hostname=~\"$host\"}", "legendFormat": "{{hostname}}"} + ], + "fieldConfig": {"defaults": {"unit": "s"}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}} + }, + + { + "id": 10, + "type": "row", + "title": "Argos's view of these instances", + "collapsed": false, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 14} + }, + { + "id": 11, + "type": "timeseries", + "title": "argos_searxng_instance_up by SearXNG instance", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 15}, + "targets": [ + {"refId": "A", "expr": "argos_searxng_instance_up", "legendFormat": "{{searxng_instance}}"} + ], + "fieldConfig": {"defaults": {"unit": "short", "min": 0, "max": 1, "custom": {"drawStyle": "line", "lineWidth": 2, "fillOpacity": 20}}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}} + }, + { + "id": 12, + "type": "timeseries", + "title": "Search latency p95 from Argos (5m)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 15}, + "targets": [ + {"refId": "A", "expr": "histogram_quantile(0.95, sum by (le, searxng_instance) (rate(argos_searxng_request_duration_seconds_bucket[5m])))", "legendFormat": "{{searxng_instance}} p95"} + ], + "fieldConfig": {"defaults": {"unit": "s"}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}} + }, + { + "id": 13, + "type": "timeseries", + "title": "Search request error ratio from Argos (5m)", + "datasource": {"type": "prometheus", "uid": "${prom}"}, + "gridPos": {"h": 8, "w": 24, "x": 0, "y": 23}, + "targets": [ + {"refId": "A", "expr": "sum by (searxng_instance) (rate(argos_searxng_requests_total{status=\"error\"}[5m])) / clamp_min(sum by (searxng_instance) (rate(argos_searxng_requests_total[5m])), 0.0001)", "legendFormat": "{{searxng_instance}}"} + ], + "fieldConfig": {"defaults": {"unit": "percentunit"}}, + "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}} + }, + + { + "id": 90, + "type": "row", + "title": "Logs", + "collapsed": false, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 31} + }, + { + "id": 91, + "type": "logs", + "title": "searxng (Loki)", + "datasource": {"type": "loki", "uid": "${loki}"}, + "gridPos": {"h": 12, "w": 24, "x": 0, "y": 32}, + "targets": [ + {"refId": "A", "expr": "{job=\"searxng\"}"} + ], + "options": {"showTime": true, "wrapLogMessage": true, "enableLogDetails": true, "dedupStrategy": "none"} + } + ] +} diff --git a/docs/ouranos.md b/docs/ouranos.md index bbf0db3..6c6b0c0 100644 --- a/docs/ouranos.md +++ b/docs/ouranos.md @@ -45,7 +45,7 @@ Each project has a number that is used to determine port numbers. - Gitea Runner (CI/CD agent) - Django Projects: Zelus (221), Angelia (222), Athena (224), Kairos (225), Icarlos (226), MCP Switchboard (227), Spelunker (228), Peitho (229), Mnemosyne (230) - FastAgent Projects: Pallas (240) -- FastAPI Projects: Daedalus (200), Arke (201) Kernos (202), Rommie (203), Orpheus (204), Periplus (205), Nike (206), Stentor (207) +- FastAPI Projects: Daedalus (200), Arke (201) Kernos (202), Rommie (203), Orpheus (204), Periplus (205), Nike (206), Stentor (207), Argos (208), ### caliban — Agent Automation @@ -101,10 +101,10 @@ Curious bridge between worlds — hosting MCP server containers. - Docker engine (API exposed on port 2375 for MCP Switchboard) - MCPO OpenAI-compatible MCP proxy 22071 -- Argos MCP Server — web search via SearXNG (port 22062) +- Argos MCP Server — web search via SearXNG (port 20861) - Grafana MCP Server (port 22063) - Neo4j MCP Server (port 22064) -- Gitea MCP Server (port 22065) +- Gitea MCP Server (port 22062) ### prospero — Observability Stack