Last active
January 23, 2026 23:04
-
-
Save chappjc/08ea5d31619a0e3971d2c1c5b4f72c9a to your computer and use it in GitHub Desktop.
pais api discovery not yet started
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| 2026-01-23 23:00:01 [debug ] NLTK_DISABLE_DOWNLOAD set: checking previous state of nltk download mode | |
| 2026-01-23 23:00:02 [info ] NLTK download set to offline mode | |
| 2026-01-23 23:00:02 [debug ] Applied thread-safety patch to NLTK's LazyCorpusLoader. | |
| 2026-01-23T23:00:16.690+00:00 [33] [139636530932608] DEBUG get_client k8s/get_client - [1mLoading in-cluster k8s-client [0m | |
| 2026-01-23T23:00:16.691+00:00 [33] [139636530932608] DEBUG get_async_client k8s/get_async_client - [1mLoading in-cluster k8s-client [0m | |
| 2026-01-23T23:00:16.691+00:00 [33] [139636530932608] DEBUG get_namespace k8s/get_namespace - [1mReading configured namespace [0m [36mnamespace[0m=[35m[in cluster][0m | |
| 2026-01-23T23:00:16.692+00:00 [33] [139636530932608] DEBUG get_namespace k8s/get_namespace - [1mUsing namespace [0m [36mnamespace[0m=[35mpais-test[0m | |
| 2026-01-23T23:00:16.692+00:00 [33] [139636530932608] DEBUG initialize_app server/initialize_app - [1mValidating k8s connectivity [0m | |
| 2026-01-23T23:00:16.711+00:00 [33] [139636530932608] DEBUG initialize_app server/initialize_app - [1mK8s connectivity successful [0m | |
| 2026-01-23T23:00:16.711+00:00 [33] [139636530932608] INFO connect_engine _base/connect_engine - [1mConnecting to DB (sync engine)[0m [36mdb_name[0m=[35mpais-db[0m [36mhost[0m=[35mpostgres[0m [36mport[0m=[35m5432[0m | |
| 2026-01-23T23:00:16.714+00:00 [33] [139636530932608] INFO async_connect_engine _base/async_connect_engine - [1mConnecting to DB (async engine)[0m [36mdb_name[0m=[35mpais-db[0m [36mhost[0m=[35mpostgres[0m [36mport[0m=[35m5432[0m | |
| 2026-01-23T23:00:16.733+00:00 [33] [139636530932608] INFO initialize_app server/initialize_app - [1mRegistering Prometheus collectors[0m | |
| 2026-01-23T23:00:16.734+00:00 [33] [139636530932608] DEBUG define_tasks _factory/define_tasks - [1mBootstrapping tasks [0m | |
| 2026-01-23T23:00:16.735+00:00 [33] [139636530932608] DEBUG initialize_app server/initialize_app - [1mSettings initialized, building API context[0m | |
| 2026-01-23T23:00:16.735+00:00 [33] [139636530932608] DEBUG celery.utils.functional functional/head_from_fun - | |
| def start_next_data_sources_indexing_task(*args, **kwargs): | |
| return 1 | |
| 2026-01-23T23:00:16.736+00:00 [33] [139636530932608] DEBUG celery.utils.functional functional/head_from_fun - | |
| def start_next_upload_indexing_task(*args, **kwargs): | |
| return 1 | |
| 2026-01-23T23:00:16.737+00:00 [33] [139636530932608] DEBUG celery.utils.functional functional/head_from_fun - | |
| def check_data_sources_indexing_finished_task(*args, **kwargs): | |
| return 1 | |
| 2026-01-23T23:00:16.738+00:00 [33] [139636530932608] DEBUG celery.utils.functional functional/head_from_fun - | |
| def check_upload_indexing_finished_task(*args, **kwargs): | |
| return 1 | |
| 2026-01-23T23:00:16.738+00:00 [33] [139636530932608] DEBUG celery.utils.functional functional/head_from_fun - | |
| def start_scheduled_indexings_task(*args, **kwargs): | |
| return 1 | |
| 2026-01-23T23:00:16.739+00:00 [33] [139636530932608] DEBUG celery.utils.functional functional/head_from_fun - | |
| def batch_clone_documents_task(*args, **kwargs): | |
| return 1 | |
| 2026-01-23T23:00:16.739+00:00 [33] [139636530932608] DEBUG celery.utils.functional functional/head_from_fun - | |
| def start_cloning_task(*args, **kwargs): | |
| return 1 | |
| 2026-01-23T23:00:16.742+00:00 [33] [139636297664192] DEBUG PaisMTlsProviderWatcher probes/run - [1mStarting watcher thread [0m [36mprobe_interval[0m=[35mdatetime.timedelta(seconds=60)[0m | |
| 2026-01-23T23:00:16.742+00:00 [33] [139636297664192] INFO PaisMTlsProviderWatcher probes/_reload_provider_certificate - [1mReloading provider certificate[0m | |
| 2026-01-23T23:00:16.742+00:00 [33] [139636297664192] DEBUG PaisMTlsProviderWatcher probes/inner - [1mLoading certificate [0m | |
| 2026-01-23T23:00:16.742+00:00 [33] [139636530932608] DEBUG initialize_app server/initialize_app - [1mBuilding API app [0m | |
| 2026-01-23T23:00:16.743+00:00 [33] [139636530932608] INFO initialize_cache_backend settings/initialize_backend - [1mUsing In Memory Cache Backend [0m | |
| 2026-01-23T23:00:16.767+00:00 [33] [139636297664192] INFO PaisMTlsProviderWatcher probes/_reload_provider_certificate - [1mProvider certificate reloaded successfully[0m [36mnum_attempts[0m=[35m1[0m | |
| 2026-01-23T23:00:16.767+00:00 [33] [139636297664192] DEBUG PaisMTlsProviderWatcher probes/_report_provider_health - [1mReporting liveness: observer alive and provider healthy[0m | |
| 2026-01-23T23:00:16.767+00:00 [33] [139636297664192] DEBUG NamedCheckinHealthProbe probes/check_in - [1mReceived check-in [0m [36mname[0m=[35mmTLS-watcher[0m [36mnow[0m=[35mdatetime.datetime(2026, 1, 23, 23, 0, 16, 767542, tzinfo=datetime.timezone.utc)[0m | |
| 2026-01-23T23:00:16.767+00:00 [33] [139636297664192] DEBUG PaisMTlsProviderWatcher probes/_report_provider_health - [1mReporting liveness: observer alive and provider healthy[0m | |
| 2026-01-23T23:00:16.772+00:00 [33] [139636297664192] DEBUG NamedCheckinHealthProbe probes/check_in - [1mReceived check-in [0m [36mname[0m=[35mmTLS-watcher[0m [36mnow[0m=[35mdatetime.datetime(2026, 1, 23, 23, 0, 16, 772820, tzinfo=datetime.timezone.utc)[0m | |
| 2026-01-23T23:00:16.773+00:00 [33] [139636297664192] DEBUG PaisMTlsProviderWatcher probes/run - [1mMonitoring directory [0m [36mpath[0m=[35mPosixPath('/mtls-cert')[0m | |
| 2026-01-23T23:00:16.773+00:00 [33] [139636297664192] DEBUG PaisMTlsProviderWatcher probes/run - [1mStarting file system watcher [0m | |
| 2026-01-23T23:00:17.185+00:00 [33] [139636530932608] INFO main _setup/setup_tracing - [1mTracing is disabled [0m | |
| 2026-01-23T23:00:17.185+00:00 [33] [139636530932608] DEBUG initialize_app server/initialize_app - [1mAPI app created successfully [0m | |
| 2026-01-23T23:00:17.185+00:00 [33] [139636530932608] INFO build_app_for_uvicorn server/build_app_for_uvicorn - [1mApp initializing complete [0m | |
| 2026-01-23T23:00:17.186+00:00 [33] [139636530932608] INFO uvicorn.error server/_serve - Started server process [33] | |
| 2026-01-23T23:00:17.186+00:00 [33] [139636530932608] INFO uvicorn.error on/startup - Waiting for application startup. | |
| 2026-01-23T23:00:17.193+00:00 [33] [139636530932608] DEBUG create_app server/lifespan - [1mLifespan setup [0m | |
| 2026-01-23T23:00:17.193+00:00 [33] [139636530932608] DEBUG create_app server/lifespan - [1mInvoking start-up hooks [0m [36mnum_hooks[0m=[35m1[0m | |
| 2026-01-23T23:00:17.193+00:00 [33] [139636530932608] DEBUG openai_compat.build __init__/lifespan_startup - [1mLifespan setup [0m | |
| 2026-01-23T23:00:17.193+00:00 [33] [139636530932608] DEBUG openai_compat.build __init__/lifespan_startup - [1mLaunching inference gateway route discovery task[0m | |
| 2026-01-23T23:00:17.194+00:00 [33] [139636530932608] DEBUG openai_compat.build __init__/lifespan_startup - [1mLifespan setup complete [0m | |
| 2026-01-23T23:00:17.194+00:00 [33] [139636530932608] DEBUG create_app server/lifespan - [1mLifespan setup complete [0m | |
| 2026-01-23T23:00:17.196+00:00 [33] [139636530932608] DEBUG AsyncCachedCredentialsRepositoryWatcher _credentials/run - [1mStarting secret watcher task [0m [36mprobe_interval[0m=[35mdatetime.timedelta(seconds=60)[0m | |
| 2026-01-23T23:00:17.243+00:00 [33] [139636530932608] DEBUG K8sRouteDiscoveryWatcher _inference_gateway_route_discovery/run - [1mStarting watch [0m [36mhave_loop_callback[0m=[35mTrue[0m [36mshutdown_poll_timeout[0m=[35mdatetime.timedelta(seconds=5)[0m | |
| 2026-01-23T23:00:17.286+00:00 [33] [139636530932608] INFO uvicorn.error on/startup - Application startup complete. | |
| 2026-01-23T23:00:17.288+00:00 [33] [139636530932608] INFO uvicorn.error server/_log_started_message - Uvicorn running on http://0.0.0.0:2080 (Press CTRL+C to quit) | |
| 2026-01-23T23:00:17.520+00:00 [33] [139636530932608] WARNING AsyncInferenceGatewayRouteRepository _base/watch - [1mWatching resources failed [0m [36mresource_type[0m=[35mInferenceGatewayRoute[0m | |
| [31m╭─[0m[31m──────────────────────────────[0m[31m [0m[1;31mTraceback [0m[1;2;31m(most recent call last)[0m[31m [0m[31m───────────────────────────────[0m[31m─╮[0m | |
| [31m│[0m [2m/usr/lib/python3.11/site-packages/paih/k8s_repositories/[0m[1m_base.py[0m:1065 in watch [31m│[0m | |
| [31m│[0m [31m│[0m | |
| [31m│[0m [2m1062 [0m[2;33m│ │ [0m[33m:raise errors.Error: Watching resources failed[0m [31m│[0m | |
| [31m│[0m [2m1063 [0m[2;33m│ │ [0m[33m"""[0m [31m│[0m | |
| [31m│[0m [2m1064 [0m[2m│ │ [0m[94mtry[0m: [31m│[0m | |
| [31m│[0m [31m❱ [0m1065 [2m│ │ │ [0m[94masync[0m [94mfor[0m event [95min[0m [96mself[0m._base_ops.watch( [31m│[0m | |
| [31m│[0m [2m1066 [0m[2m│ │ │ │ [0mlabel_selector=label_selector, [31m│[0m | |
| [31m│[0m [2m1067 [0m[2m│ │ │ │ [0mtimeout=timeout, [31m│[0m | |
| [31m│[0m [2m1068 [0m[2m│ │ │ │ [0mresource_version=resource_version, [31m│[0m | |
| [31m│[0m [31m│[0m | |
| [31m│[0m [34m╭─[0m[34m──────────────────────────────────────────[0m[34m locals [0m[34m──────────────────────────────────────────[0m[34m─╮[0m [31m│[0m | |
| [31m│[0m [34m│[0m [3;33merr[0m[31m =[0m [1;35mApiException[0m[1m([0m[1m)[0m [34m│[0m [31m│[0m | |
| [31m│[0m [34m│[0m [3;33mlabel_selector[0m[31m =[0m [3;35mNone[0m [34m│[0m [31m│[0m | |
| [31m│[0m [34m│[0m [3;33mresource_version[0m[31m =[0m [3;35mNone[0m [34m│[0m [31m│[0m | |
| [31m│[0m [34m│[0m [3;33mself[0m[31m =[0m [1m<[0m[1;95mpaih.k8s_repositories._inference_gateway_routes.AsyncInferenceGatewayRo…[0m [34m│[0m [31m│[0m | |
| [31m│[0m [34m│[0m [39mobject at [0m[1;36m0x7eff8d513790[0m[1m>[0m [34m│[0m [31m│[0m | |
| [31m│[0m [34m│[0m [3;33mtimeout[0m[31m =[0m [1;35mdatetime.timedelta[0m[1m([0m[33mmicroseconds[0m=[1;36m100000[0m[1m)[0m [34m│[0m [31m│[0m | |
| [31m│[0m [34m╰──────────────────────────────────────────────────────────────────────────────────────────────╯[0m [31m│[0m | |
| [31m│[0m [31m│[0m | |
| [31m│[0m [2m/usr/lib/python3.11/site-packages/paih/k8s_repositories/[0m[1m_base.py[0m:321 in watch [31m│[0m | |
| [31m│[0m [31m│[0m | |
| [31m│[0m [2m 318 [0m[2;33m│ │ [0m[33m"""[0m [31m│[0m | |
| [31m│[0m [2m 319 [0m[2m│ │ [0m[2m# see comment on `self._watch_stream()`: we explicitly iterate over the events[0m [31m│[0m | |
| [31m│[0m [2m 320 [0m[2m│ │ [0m[2m# to indicate to the type checker that this function is indeed an async generato[0m [31m│[0m | |
| [31m│[0m [31m❱ [0m 321 [2m│ │ [0m[94masync[0m [94mfor[0m event [95min[0m [1;4;96mself[0m[1;4m._watch_stream([0m [31m│[0m | |
| [31m│[0m [2m 322 [0m[2m│ │ │ [0m[1;4mwatch=kubernetes_asyncio.watch.Watch(),[0m [31m│[0m | |
| [31m│[0m [2m 323 [0m[2m│ │ │ [0m[1;4mlabel_selector=label_selector,[0m [31m│[0m | |
| [31m│[0m [2m 324 [0m[2m│ │ │ [0m[1;4mresource_version=resource_version,[0m [31m│[0m | |
| [31m│[0m [31m│[0m | |
| [31m│[0m [34m╭─[0m[34m──────────────────────────────────────────[0m[34m locals [0m[34m──────────────────────────────────────────[0m[34m─╮[0m [31m│[0m | |
| [31m│[0m [34m│[0m [3;33mlabel_selector[0m[31m =[0m [3;35mNone[0m [34m│[0m [31m│[0m | |
| [31m│[0m [34m│[0m [3;33mresource_version[0m[31m =[0m [3;35mNone[0m [34m│[0m [31m│[0m | |
| [31m│[0m [34m│[0m [3;33mself[0m[31m =[0m [1m<[0m[1;95mpaih.k8s_repositories._inference_gateway_routes.AsyncInferenceGatewayRo…[0m [34m│[0m [31m│[0m | |
| [31m│[0m [34m│[0m [39mobject at [0m[1;36m0x7eff8d23d690[0m[1m>[0m [34m│[0m [31m│[0m | |
| [31m│[0m [34m│[0m [3;33mtimeout[0m[31m =[0m [1;35mdatetime.timedelta[0m[1m([0m[33mmicroseconds[0m=[1;36m100000[0m[1m)[0m [34m│[0m [31m│[0m | |
| [31m│[0m [34m╰──────────────────────────────────────────────────────────────────────────────────────────────╯[0m [31m│[0m | |
| [31m│[0m [31m│[0m | |
| [31m│[0m [2m/usr/lib/python3.11/site-packages/paih/k8s_repositories/[0m[1m_base.py[0m:585 in _watch_stream [31m│[0m | |
| [31m│[0m [31m│[0m | |
| [31m│[0m [2m 582 [0m[2m│ │ [0mresource_version: Optional[k8s_schemas.ResourceVersion] = [94mNone[0m, [31m│[0m | |
| [31m│[0m [2m 583 [0m[2m│ [0m) -> AsyncIterator[ResourceWatchEvent[k8s_schemas.ResourceType]]: [31m│[0m | |
| [31m│[0m [2m 584 [0m[2m│ │ [0m[94masync[0m [94mwith[0m [96mself[0m._k8s_client.get_custom_objects_client() [94mas[0m client: [31m│[0m | |
| [31m│[0m [31m❱ [0m 585 [2m│ │ │ [0m[94masync[0m [94mfor[0m event [95min[0m [1;4mwatch.stream([0m [31m│[0m | |
| [31m│[0m [2m 586 [0m[2m│ │ │ │ [0m[1;4mfunc=client.list_namespaced_custom_object,[0m [31m│[0m | |
| [31m│[0m [2m 587 [0m[2m│ │ │ │ [0m[1;4mgroup=[0m[1;4;96mself[0m[1;4m._api_group,[0m [31m│[0m | |
| [31m│[0m [2m 588 [0m[2m│ │ │ │ [0m[1;4mversion=[0m[1;4;96mself[0m[1;4m._api_version,[0m [31m│[0m | |
| [31m│[0m [31m│[0m | |
| [31m│[0m [34m╭─[0m[34m──────────────────────────────────────────[0m[34m locals [0m[34m──────────────────────────────────────────[0m[34m─╮[0m [31m│[0m | |
| [31m│[0m [34m│[0m [3;33mclient[0m[31m =[0m [1m<[0m[1;95mkubernetes_asyncio.client.api.custom_objects_api.CustomObjectsApi[0m[39m object[0m [34m│[0m [31m│[0m | |
| [31m│[0m [34m│[0m [39mat [0m[1;36m0x7eff8720dfd0[0m[1m>[0m [34m│[0m [31m│[0m | |
| [31m│[0m [34m│[0m [3;33mlabel_selector[0m[31m =[0m [3;35mNone[0m [34m│[0m [31m│[0m | |
| [31m│[0m [34m│[0m [3;33mresource_version[0m[31m =[0m [3;35mNone[0m [34m│[0m [31m│[0m | |
| [31m│[0m [34m│[0m [3;33mself[0m[31m =[0m [1m<[0m[1;95mpaih.k8s_repositories._inference_gateway_routes.AsyncInferenceGatewayRo…[0m [34m│[0m [31m│[0m | |
| [31m│[0m [34m│[0m [39mobject at [0m[1;36m0x7eff8d23d690[0m[1m>[0m [34m│[0m [31m│[0m | |
| [31m│[0m [34m│[0m [3;33mtimeout_seconds[0m[31m =[0m [1;36m1[0m [34m│[0m [31m│[0m | |
| [31m│[0m [34m│[0m [3;33mwatch[0m[31m =[0m [1m<[0m[1;95mkubernetes_asyncio.watch.watch.Watch[0m[39m object at [0m[1;36m0x7eff875a61d0[0m[1m>[0m [34m│[0m [31m│[0m | |
| [31m│[0m [34m╰──────────────────────────────────────────────────────────────────────────────────────────────╯[0m [31m│[0m | |
| [31m│[0m [31m│[0m | |
| [31m│[0m [2m/usr/lib/python3.11/site-packages/kubernetes_asyncio/watch/[0m[1mwatch.py[0m:133 in __anext__ [31m│[0m | |
| [31m│[0m [31m│[0m | |
| [31m│[0m [2m130 [0m[2m│ [0m [31m│[0m | |
| [31m│[0m [2m131 [0m[2m│ [0m[94masync[0m [94mdef[0m[90m [0m[92m__anext__[0m([96mself[0m): [31m│[0m | |
| [31m│[0m [2m132 [0m[2m│ │ [0m[94mtry[0m: [31m│[0m | |
| [31m│[0m [31m❱ [0m133 [2m│ │ │ [0m[94mreturn[0m [94mawait[0m [96mself[0m.next() [31m│[0m | |
| [31m│[0m [2m134 [0m[2m│ │ [0m[94mexcept[0m: [2m# noqa: E722[0m [31m│[0m | |
| [31m│[0m [2m135 [0m[2m│ │ │ [0m[94mawait[0m [96mself[0m.close() [31m│[0m | |
| [31m│[0m [2m136 [0m[2m│ │ │ [0m[1;4;94mraise[0m [31m│[0m | |
| [31m│[0m [31m│[0m | |
| [31m│[0m [34m╭─[0m[34m───────────────────────────────[0m[34m locals [0m[34m───────────────────────────────[0m[34m─╮[0m [31m│[0m | |
| [31m│[0m [34m│[0m [3;33mself[0m[31m =[0m [1m<[0m[1;95mkubernetes_asyncio.watch.watch.Watch[0m[39m object at [0m[1;36m0x7eff875a61d0[0m[1m>[0m [34m│[0m [31m│[0m | |
| [31m│[0m [34m╰────────────────────────────────────────────────────────────────────────╯[0m [31m│[0m | |
| [31m│[0m [31m│[0m | |
| [31m│[0m [2m/usr/lib/python3.11/site-packages/kubernetes_asyncio/watch/[0m[1mwatch.py[0m:176 in next [31m│[0m | |
| [31m│[0m [31m│[0m | |
| [31m│[0m [2m173 [0m[2m│ │ │ [0m[94mif[0m [96mself[0m.return_type == [33m'[0m[33mstr[0m[33m'[0m: [31m│[0m | |
| [31m│[0m [2m174 [0m[2m│ │ │ │ [0m[94mreturn[0m line [31m│[0m | |
| [31m│[0m [2m175 [0m[2m│ │ │ [0m [31m│[0m | |
| [31m│[0m [31m❱ [0m176 [2m│ │ │ [0m[94mreturn[0m [1;4;96mself[0m[1;4m.unmarshal_event(line, [0m[1;4;96mself[0m[1;4m.return_type)[0m [31m│[0m | |
| [31m│[0m [2m177 [0m[2m│ [0m [31m│[0m | |
| [31m│[0m [2m178 [0m[2m│ [0m[94mdef[0m[90m [0m[92mstream[0m([96mself[0m, func, *args, **kwargs): [31m│[0m | |
| [31m│[0m [2m179 [0m[2;90m│ │ [0m[33m"""Watch an API resource and stream the result back via a generator.[0m [31m│[0m | |
| [31m│[0m [31m│[0m | |
| [31m│[0m [34m╭─[0m[34m──────────────────────────────────────────[0m[34m locals [0m[34m──────────────────────────────────────────[0m[34m─╮[0m [31m│[0m | |
| [31m│[0m [34m│[0m [3;33mline[0m[31m =[0m [32m'[0m[32m{[0m[32m"kind":"Status","apiVersion":"v1","metadata":[0m[32m{[0m[32m}[0m[32m,"status":"Failure","message":"s'[0m+[1;36m1…[0m [34m│[0m [31m│[0m | |
| [31m│[0m [34m│[0m [3;33mself[0m[31m =[0m [1m<[0m[1;95mkubernetes_asyncio.watch.watch.Watch[0m[39m object at [0m[1;36m0x7eff875a61d0[0m[1m>[0m [34m│[0m [31m│[0m | |
| [31m│[0m [34m╰──────────────────────────────────────────────────────────────────────────────────────────────╯[0m [31m│[0m | |
| [31m│[0m [31m│[0m | |
| [31m│[0m [2m/usr/lib/python3.11/site-packages/kubernetes_asyncio/watch/[0m[1mwatch.py[0m:87 in unmarshal_event [31m│[0m | |
| [31m│[0m [31m│[0m | |
| [31m│[0m [2m 84 [0m[2m│ │ │ [0m[2m# raise error with code if set[0m [31m│[0m | |
| [31m│[0m [2m 85 [0m[2m│ │ │ [0m[94mif[0m [33m'[0m[33mcode[0m[33m'[0m [95min[0m js: [31m│[0m | |
| [31m│[0m [2m 86 [0m[2m│ │ │ │ [0mreason = [33m"[0m[33m{}[0m[33m: [0m[33m{}[0m[33m"[0m.format(js.get([33m'[0m[33mreason[0m[33m'[0m), js.get([33m'[0m[33mmessage[0m[33m'[0m)) [31m│[0m | |
| [31m│[0m [31m❱ [0m 87 [2m│ │ │ │ [0m[1;4;94mraise[0m[1;4m client.exceptions.ApiException(status=js[[0m[1;4;33m'[0m[1;4;33mcode[0m[1;4;33m'[0m[1;4m], reason=reason)[0m [31m│[0m | |
| [31m│[0m [2m 88 [0m[2m│ │ │ [0m [31m│[0m | |
| [31m│[0m [2m 89 [0m[2m│ │ │ [0m[94mraise[0m [96mException[0m(([33m"[0m[33mMalformed JSON response, the [0m[33m'[0m[33mobject[0m[33m'[0m[33m and/or [0m[33m"[0m [31m│[0m | |
| [31m│[0m [2m 90 [0m[2m│ │ │ │ │ │ │ [0m[33m"[0m[33m'[0m[33mtype[0m[33m'[0m[33m field is missing. JSON: [0m[33m{}[0m[33m"[0m).format(js)) [31m│[0m | |
| [31m│[0m [31m│[0m | |
| [31m│[0m [34m╭─[0m[34m──────────────────────────────────────────[0m[34m locals [0m[34m──────────────────────────────────────────[0m[34m─╮[0m [31m│[0m | |
| [31m│[0m [34m│[0m [3;33mdata[0m[31m =[0m [32m'[0m[32m{[0m[32m"kind":"Status","apiVersion":"v1","metadata":[0m[32m{[0m[32m}[0m[32m,"status":"Failure","messa…[0m [34m│[0m [31m│[0m | |
| [31m│[0m [34m│[0m [3;33mjs[0m[31m =[0m [1m{[0m [34m│[0m [31m│[0m | |
| [31m│[0m [34m│[0m [2;32m│ [0m[32m'kind'[0m: [32m'Status'[0m, [34m│[0m [31m│[0m | |
| [31m│[0m [34m│[0m [2;32m│ [0m[32m'apiVersion'[0m: [32m'v1'[0m, [34m│[0m [31m│[0m | |
| [31m│[0m [34m│[0m [2;32m│ [0m[32m'metadata'[0m: [1m{[0m[1m}[0m, [34m│[0m [31m│[0m | |
| [31m│[0m [34m│[0m [2;32m│ [0m[32m'status'[0m: [32m'Failure'[0m, [34m│[0m [31m│[0m | |
| [31m│[0m [34m│[0m [2;32m│ [0m[32m'message'[0m: [32m'storage is [0m[32m([0m[32mre[0m[32m)[0m[32minitializing'[0m, [34m│[0m [31m│[0m | |
| [31m│[0m [34m│[0m [2;32m│ [0m[32m'reason'[0m: [32m'TooManyRequests'[0m, [34m│[0m [31m│[0m | |
| [31m│[0m [34m│[0m [2;32m│ [0m[32m'details'[0m: [1m{[0m[32m'retryAfterSeconds'[0m: [1;36m1[0m[1m}[0m, [34m│[0m [31m│[0m | |
| [31m│[0m [34m│[0m [2;32m│ [0m[32m'code'[0m: [1;36m429[0m [34m│[0m [31m│[0m | |
| [31m│[0m [34m│[0m [1m}[0m [34m│[0m [31m│[0m | |
| [31m│[0m [34m│[0m [3;33mreason[0m[31m =[0m [32m'TooManyRequests: storage is [0m[32m([0m[32mre[0m[32m)[0m[32minitializing'[0m [34m│[0m [31m│[0m | |
| [31m│[0m [34m│[0m [3;33mresponse_type[0m[31m =[0m [32m'object'[0m [34m│[0m [31m│[0m | |
| [31m│[0m [34m│[0m [3;33mself[0m[31m =[0m [1m<[0m[1;95mkubernetes_asyncio.watch.watch.Watch[0m[39m object at [0m[1;36m0x7eff875a61d0[0m[1m>[0m [34m│[0m [31m│[0m | |
| [31m│[0m [34m╰──────────────────────────────────────────────────────────────────────────────────────────────╯[0m [31m│[0m | |
| [31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯[0m | |
| [1;91mApiException: [0m[1m([0m[1;36m429[0m[1m)[0m | |
| Reason: TooManyRequests: storage is [1m([0mre[1m)[0minitializing | |
| 2026-01-23T23:00:17.522+00:00 [33] [139636530932608] DEBUG AsyncCachedCredentialsRepositoryWatcher _credentials/run - [1mProcessing event [0m [36mevent_type[0m=[35mADDED[0m [36mresource_name[0m=[35mllm-service-token[0m | |
| 2026-01-23T23:00:17.522+00:00 [33] [139636530932608] DEBUG NamedCheckinHealthProbe probes/check_in - [1mReceived check-in [0m [36mname[0m=[35minference-gateway-secret-watcher[0m [36mnow[0m=[35mdatetime.datetime(2026, 1, 23, 23, 0, 17, 522876, tzinfo=datetime.timezone.utc)[0m | |
| 2026-01-23T23:00:18.336+00:00 [33] [139636530932608] DEBUG AsyncCachedCredentialsRepositoryWatcher _credentials/run - [1mReporting liveness: no events received[0m | |
| 2026-01-23T23:00:18.336+00:00 [33] [139636530932608] DEBUG NamedCheckinHealthProbe probes/check_in - [1mReceived check-in [0m [36mname[0m=[35minference-gateway-secret-watcher[0m [36mnow[0m=[35mdatetime.datetime(2026, 1, 23, 23, 0, 18, 336807, tzinfo=datetime.timezone.utc)[0m | |
| 2026-01-23T23:00:18.395+00:00 [33] [139636530932608] DEBUG AsyncCachedCredentialsRepositoryWatcher _credentials/run - [1mProcessing event [0m [36mevent_type[0m=[35mADDED[0m [36mresource_name[0m=[35mllm-service-token[0m | |
| 2026-01-23T23:00:18.402+00:00 [33] [139636530932608] DEBUG NamedCheckinHealthProbe probes/check_in - [1mReceived check-in [0m [36mname[0m=[35minference-gateway-secret-watcher[0m [36mnow[0m=[35mdatetime.datetime(2026, 1, 23, 23, 0, 18, 402057, tzinfo=datetime.timezone.utc)[0m | |
| 2026-01-23T23:00:20.912+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m9cdff34f0d274a3783593ca7f9765980[0m | |
| 2026-01-23T23:00:20.912+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35m9cdff34f0d274a3783593ca7f9765980[0m | |
| 2026-01-23T23:00:21.077+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:00:21.077+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:00:21.180+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m9cdff34f0d274a3783593ca7f9765980[0m | |
| 2026-01-23T23:00:21.180+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35m9cdff34f0d274a3783593ca7f9765980[0m | |
| 2026-01-23T23:00:21.674+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:00:21.675+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:00:21.685+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m9cdff34f0d274a3783593ca7f9765980[0m | |
| 2026-01-23T23:00:21.686+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35m9cdff34f0d274a3783593ca7f9765980[0m | |
| 2026-01-23T23:00:21.686+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 22, 59, 51, 686816, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m9cdff34f0d274a3783593ca7f9765980[0m | |
| 2026-01-23T23:00:21.687+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m9cdff34f0d274a3783593ca7f9765980[0m | |
| 2026-01-23T23:00:21.687+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35m9cdff34f0d274a3783593ca7f9765980[0m | |
| 2026-01-23T23:00:21.687+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m9cdff34f0d274a3783593ca7f9765980[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:00:21.688+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:54800 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:00:26.690+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m1acabe55a2ee4ec19c98ab8b5682f376[0m | |
| 2026-01-23T23:00:26.690+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35m1acabe55a2ee4ec19c98ab8b5682f376[0m | |
| 2026-01-23T23:00:26.696+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:00:26.696+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:00:26.704+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m1acabe55a2ee4ec19c98ab8b5682f376[0m | |
| 2026-01-23T23:00:26.704+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35m1acabe55a2ee4ec19c98ab8b5682f376[0m | |
| 2026-01-23T23:00:26.710+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:00:26.710+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:00:26.716+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m1acabe55a2ee4ec19c98ab8b5682f376[0m | |
| 2026-01-23T23:00:26.716+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35m1acabe55a2ee4ec19c98ab8b5682f376[0m | |
| 2026-01-23T23:00:26.716+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 22, 59, 56, 716536, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m1acabe55a2ee4ec19c98ab8b5682f376[0m | |
| 2026-01-23T23:00:26.716+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m1acabe55a2ee4ec19c98ab8b5682f376[0m | |
| 2026-01-23T23:00:26.716+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35m1acabe55a2ee4ec19c98ab8b5682f376[0m | |
| 2026-01-23T23:00:26.717+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m1acabe55a2ee4ec19c98ab8b5682f376[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:00:26.717+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:54814 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:00:31.719+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35mbc22890c146c4bff83e8cbae45a6ce9f[0m | |
| 2026-01-23T23:00:31.719+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35mbc22890c146c4bff83e8cbae45a6ce9f[0m | |
| 2026-01-23T23:00:31.723+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:00:31.723+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:00:31.730+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35mbc22890c146c4bff83e8cbae45a6ce9f[0m | |
| 2026-01-23T23:00:31.731+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35mbc22890c146c4bff83e8cbae45a6ce9f[0m | |
| 2026-01-23T23:00:31.737+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:00:31.737+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:00:31.743+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35mbc22890c146c4bff83e8cbae45a6ce9f[0m | |
| 2026-01-23T23:00:31.743+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35mbc22890c146c4bff83e8cbae45a6ce9f[0m | |
| 2026-01-23T23:00:31.743+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 0, 1, 743560, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35mbc22890c146c4bff83e8cbae45a6ce9f[0m | |
| 2026-01-23T23:00:31.743+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35mbc22890c146c4bff83e8cbae45a6ce9f[0m | |
| 2026-01-23T23:00:31.744+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35mbc22890c146c4bff83e8cbae45a6ce9f[0m | |
| 2026-01-23T23:00:31.744+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35mbc22890c146c4bff83e8cbae45a6ce9f[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:00:31.744+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:33950 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:00:36.746+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m375a5de56d42418594f5be98441fc6fe[0m | |
| 2026-01-23T23:00:36.747+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35m375a5de56d42418594f5be98441fc6fe[0m | |
| 2026-01-23T23:00:36.758+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:00:36.758+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:00:36.765+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m375a5de56d42418594f5be98441fc6fe[0m | |
| 2026-01-23T23:00:36.766+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35m375a5de56d42418594f5be98441fc6fe[0m | |
| 2026-01-23T23:00:36.773+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:00:36.773+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:00:36.779+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m375a5de56d42418594f5be98441fc6fe[0m | |
| 2026-01-23T23:00:36.779+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35m375a5de56d42418594f5be98441fc6fe[0m | |
| 2026-01-23T23:00:36.780+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 0, 6, 780205, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m375a5de56d42418594f5be98441fc6fe[0m | |
| 2026-01-23T23:00:36.780+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m375a5de56d42418594f5be98441fc6fe[0m | |
| 2026-01-23T23:00:36.780+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35m375a5de56d42418594f5be98441fc6fe[0m | |
| 2026-01-23T23:00:36.781+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m375a5de56d42418594f5be98441fc6fe[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:00:36.781+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:33958 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:00:41.784+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m97e9c9be0a2f4112b398cce8a03962c4[0m | |
| 2026-01-23T23:00:41.785+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35m97e9c9be0a2f4112b398cce8a03962c4[0m | |
| 2026-01-23T23:00:41.789+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:00:41.789+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:00:41.797+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m97e9c9be0a2f4112b398cce8a03962c4[0m | |
| 2026-01-23T23:00:41.797+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35m97e9c9be0a2f4112b398cce8a03962c4[0m | |
| 2026-01-23T23:00:41.806+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:00:41.806+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:00:41.812+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m97e9c9be0a2f4112b398cce8a03962c4[0m | |
| 2026-01-23T23:00:41.812+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35m97e9c9be0a2f4112b398cce8a03962c4[0m | |
| 2026-01-23T23:00:41.812+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 0, 11, 812816, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m97e9c9be0a2f4112b398cce8a03962c4[0m | |
| 2026-01-23T23:00:41.812+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m97e9c9be0a2f4112b398cce8a03962c4[0m | |
| 2026-01-23T23:00:41.813+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35m97e9c9be0a2f4112b398cce8a03962c4[0m | |
| 2026-01-23T23:00:41.813+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m97e9c9be0a2f4112b398cce8a03962c4[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:00:41.813+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:52892 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:00:46.815+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35mefa46f3267554fc3910040e932ce9ee1[0m | |
| 2026-01-23T23:00:46.816+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35mefa46f3267554fc3910040e932ce9ee1[0m | |
| 2026-01-23T23:00:46.820+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:00:46.820+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:00:46.828+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35mefa46f3267554fc3910040e932ce9ee1[0m | |
| 2026-01-23T23:00:46.829+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35mefa46f3267554fc3910040e932ce9ee1[0m | |
| 2026-01-23T23:00:46.836+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:00:46.836+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:00:46.843+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35mefa46f3267554fc3910040e932ce9ee1[0m | |
| 2026-01-23T23:00:46.844+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35mefa46f3267554fc3910040e932ce9ee1[0m | |
| 2026-01-23T23:00:46.844+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 0, 16, 844487, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35mefa46f3267554fc3910040e932ce9ee1[0m | |
| 2026-01-23T23:00:46.844+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35mefa46f3267554fc3910040e932ce9ee1[0m | |
| 2026-01-23T23:00:46.845+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35mefa46f3267554fc3910040e932ce9ee1[0m | |
| 2026-01-23T23:00:46.845+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35mefa46f3267554fc3910040e932ce9ee1[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:00:46.845+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:52906 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:00:51.852+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35mbd7f3b6dc2c24119a79f8eb15459c17b[0m | |
| 2026-01-23T23:00:51.853+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35mbd7f3b6dc2c24119a79f8eb15459c17b[0m | |
| 2026-01-23T23:00:51.864+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:00:51.865+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:00:51.877+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35mbd7f3b6dc2c24119a79f8eb15459c17b[0m | |
| 2026-01-23T23:00:51.877+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35mbd7f3b6dc2c24119a79f8eb15459c17b[0m | |
| 2026-01-23T23:00:51.887+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:00:51.887+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:00:51.893+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35mbd7f3b6dc2c24119a79f8eb15459c17b[0m | |
| 2026-01-23T23:00:51.893+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35mbd7f3b6dc2c24119a79f8eb15459c17b[0m | |
| 2026-01-23T23:00:51.894+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 0, 21, 893879, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35mbd7f3b6dc2c24119a79f8eb15459c17b[0m | |
| 2026-01-23T23:00:51.894+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35mbd7f3b6dc2c24119a79f8eb15459c17b[0m | |
| 2026-01-23T23:00:51.894+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35mbd7f3b6dc2c24119a79f8eb15459c17b[0m | |
| 2026-01-23T23:00:51.895+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35mbd7f3b6dc2c24119a79f8eb15459c17b[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:00:51.895+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:42962 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:00:56.897+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m20da39ca9eec4a55a8de30262e8babb2[0m | |
| 2026-01-23T23:00:56.898+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35m20da39ca9eec4a55a8de30262e8babb2[0m | |
| 2026-01-23T23:00:56.902+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:00:56.902+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:00:56.909+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m20da39ca9eec4a55a8de30262e8babb2[0m | |
| 2026-01-23T23:00:56.909+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35m20da39ca9eec4a55a8de30262e8babb2[0m | |
| 2026-01-23T23:00:56.916+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:00:56.916+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:00:56.922+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m20da39ca9eec4a55a8de30262e8babb2[0m | |
| 2026-01-23T23:00:56.922+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35m20da39ca9eec4a55a8de30262e8babb2[0m | |
| 2026-01-23T23:00:56.923+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 0, 26, 923035, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m20da39ca9eec4a55a8de30262e8babb2[0m | |
| 2026-01-23T23:00:56.923+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m20da39ca9eec4a55a8de30262e8babb2[0m | |
| 2026-01-23T23:00:56.923+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35m20da39ca9eec4a55a8de30262e8babb2[0m | |
| 2026-01-23T23:00:56.923+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m20da39ca9eec4a55a8de30262e8babb2[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:00:56.924+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:42978 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:01:01.926+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35me0639e9db6814861a3e6f18d759ade1a[0m | |
| 2026-01-23T23:01:01.927+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35me0639e9db6814861a3e6f18d759ade1a[0m | |
| 2026-01-23T23:01:01.931+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:01:01.931+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:01:01.946+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35me0639e9db6814861a3e6f18d759ade1a[0m | |
| 2026-01-23T23:01:01.946+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35me0639e9db6814861a3e6f18d759ade1a[0m | |
| 2026-01-23T23:01:01.955+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:01:01.955+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:01:01.968+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35me0639e9db6814861a3e6f18d759ade1a[0m | |
| 2026-01-23T23:01:01.968+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35me0639e9db6814861a3e6f18d759ade1a[0m | |
| 2026-01-23T23:01:01.968+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 0, 31, 968816, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35me0639e9db6814861a3e6f18d759ade1a[0m | |
| 2026-01-23T23:01:01.969+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35me0639e9db6814861a3e6f18d759ade1a[0m | |
| 2026-01-23T23:01:01.969+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35me0639e9db6814861a3e6f18d759ade1a[0m | |
| 2026-01-23T23:01:01.970+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35me0639e9db6814861a3e6f18d759ade1a[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:01:01.970+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:41830 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:01:03.757+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/metrics[0m [36mrequest_id[0m=[35mfdb9d7518c0f4510849172a741f1b114[0m | |
| 2026-01-23T23:01:03.865+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/metrics[0m [36mrequest_id[0m=[35mfdb9d7518c0f4510849172a741f1b114[0m [36mstatus_code[0m=[35m200[0m | |
| 2026-01-23T23:01:03.865+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 127.0.0.1:57364 - "GET /metrics HTTP/1.1" 200 | |
| 2026-01-23T23:01:06.972+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35mf62f86787f074bd292f54f10f9ad6071[0m | |
| 2026-01-23T23:01:06.973+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35mf62f86787f074bd292f54f10f9ad6071[0m | |
| 2026-01-23T23:01:06.984+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:01:06.984+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:01:06.998+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35mf62f86787f074bd292f54f10f9ad6071[0m | |
| 2026-01-23T23:01:06.998+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35mf62f86787f074bd292f54f10f9ad6071[0m | |
| 2026-01-23T23:01:07.009+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:01:07.009+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:01:07.019+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35mf62f86787f074bd292f54f10f9ad6071[0m | |
| 2026-01-23T23:01:07.019+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35mf62f86787f074bd292f54f10f9ad6071[0m | |
| 2026-01-23T23:01:07.019+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 0, 37, 19366, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35mf62f86787f074bd292f54f10f9ad6071[0m | |
| 2026-01-23T23:01:07.019+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35mf62f86787f074bd292f54f10f9ad6071[0m | |
| 2026-01-23T23:01:07.019+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35mf62f86787f074bd292f54f10f9ad6071[0m | |
| 2026-01-23T23:01:07.019+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35mf62f86787f074bd292f54f10f9ad6071[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:01:07.020+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:41846 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:01:12.021+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m05d6a8d729dd46d9a1cc4eaf99475ef0[0m | |
| 2026-01-23T23:01:12.023+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35m05d6a8d729dd46d9a1cc4eaf99475ef0[0m | |
| 2026-01-23T23:01:12.027+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:01:12.027+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:01:12.035+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m05d6a8d729dd46d9a1cc4eaf99475ef0[0m | |
| 2026-01-23T23:01:12.035+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35m05d6a8d729dd46d9a1cc4eaf99475ef0[0m | |
| 2026-01-23T23:01:12.043+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:01:12.043+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:01:12.049+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m05d6a8d729dd46d9a1cc4eaf99475ef0[0m | |
| 2026-01-23T23:01:12.050+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35m05d6a8d729dd46d9a1cc4eaf99475ef0[0m | |
| 2026-01-23T23:01:12.050+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 0, 42, 50226, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m05d6a8d729dd46d9a1cc4eaf99475ef0[0m | |
| 2026-01-23T23:01:12.050+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m05d6a8d729dd46d9a1cc4eaf99475ef0[0m | |
| 2026-01-23T23:01:12.050+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35m05d6a8d729dd46d9a1cc4eaf99475ef0[0m | |
| 2026-01-23T23:01:12.050+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m05d6a8d729dd46d9a1cc4eaf99475ef0[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:01:12.050+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:55902 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:01:16.862+00:00 [33] [139636297664192] DEBUG PaisMTlsProviderWatcher probes/_report_provider_health - [1mReporting liveness: observer alive and provider healthy[0m | |
| 2026-01-23T23:01:16.862+00:00 [33] [139636297664192] DEBUG NamedCheckinHealthProbe probes/check_in - [1mReceived check-in [0m [36mname[0m=[35mmTLS-watcher[0m [36mnow[0m=[35mdatetime.datetime(2026, 1, 23, 23, 1, 16, 862781, tzinfo=datetime.timezone.utc)[0m | |
| 2026-01-23T23:01:17.052+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35md34f9ea431e04fa288bd7025f3a47b59[0m | |
| 2026-01-23T23:01:17.052+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35md34f9ea431e04fa288bd7025f3a47b59[0m | |
| 2026-01-23T23:01:17.057+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:01:17.058+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:01:17.066+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35md34f9ea431e04fa288bd7025f3a47b59[0m | |
| 2026-01-23T23:01:17.066+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35md34f9ea431e04fa288bd7025f3a47b59[0m | |
| 2026-01-23T23:01:17.073+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:01:17.073+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:01:17.079+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35md34f9ea431e04fa288bd7025f3a47b59[0m | |
| 2026-01-23T23:01:17.079+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35md34f9ea431e04fa288bd7025f3a47b59[0m | |
| 2026-01-23T23:01:17.079+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 0, 47, 79305, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35md34f9ea431e04fa288bd7025f3a47b59[0m | |
| 2026-01-23T23:01:17.079+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35md34f9ea431e04fa288bd7025f3a47b59[0m | |
| 2026-01-23T23:01:17.079+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35md34f9ea431e04fa288bd7025f3a47b59[0m | |
| 2026-01-23T23:01:17.079+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35md34f9ea431e04fa288bd7025f3a47b59[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:01:17.079+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:55908 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:01:18.395+00:00 [33] [139636530932608] DEBUG AsyncCachedCredentialsRepositoryWatcher _credentials/run - [1mReporting liveness: no events received[0m | |
| 2026-01-23T23:01:18.395+00:00 [33] [139636530932608] DEBUG NamedCheckinHealthProbe probes/check_in - [1mReceived check-in [0m [36mname[0m=[35minference-gateway-secret-watcher[0m [36mnow[0m=[35mdatetime.datetime(2026, 1, 23, 23, 1, 18, 395550, tzinfo=datetime.timezone.utc)[0m | |
| 2026-01-23T23:01:18.485+00:00 [33] [139636530932608] DEBUG AsyncCachedCredentialsRepositoryWatcher _credentials/run - [1mProcessing event [0m [36mevent_type[0m=[35mADDED[0m [36mresource_name[0m=[35mllm-service-token[0m | |
| 2026-01-23T23:01:18.485+00:00 [33] [139636530932608] DEBUG NamedCheckinHealthProbe probes/check_in - [1mReceived check-in [0m [36mname[0m=[35minference-gateway-secret-watcher[0m [36mnow[0m=[35mdatetime.datetime(2026, 1, 23, 23, 1, 18, 485299, tzinfo=datetime.timezone.utc)[0m | |
| 2026-01-23T23:01:22.081+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m44c87ef204814839a246e65e526239be[0m | |
| 2026-01-23T23:01:22.081+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35m44c87ef204814839a246e65e526239be[0m | |
| 2026-01-23T23:01:22.086+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:01:22.087+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:01:22.094+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m44c87ef204814839a246e65e526239be[0m | |
| 2026-01-23T23:01:22.094+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35m44c87ef204814839a246e65e526239be[0m | |
| 2026-01-23T23:01:22.102+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:01:22.103+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:01:22.110+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m44c87ef204814839a246e65e526239be[0m | |
| 2026-01-23T23:01:22.111+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35m44c87ef204814839a246e65e526239be[0m | |
| 2026-01-23T23:01:22.111+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 0, 52, 111202, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m44c87ef204814839a246e65e526239be[0m | |
| 2026-01-23T23:01:22.111+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m44c87ef204814839a246e65e526239be[0m | |
| 2026-01-23T23:01:22.111+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35m44c87ef204814839a246e65e526239be[0m | |
| 2026-01-23T23:01:22.112+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m44c87ef204814839a246e65e526239be[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:01:22.112+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:41312 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:01:27.116+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m1dbf7c04e9394591ad2d1e9d747c1d69[0m | |
| 2026-01-23T23:01:27.116+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35m1dbf7c04e9394591ad2d1e9d747c1d69[0m | |
| 2026-01-23T23:01:27.120+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:01:27.120+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:01:27.127+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m1dbf7c04e9394591ad2d1e9d747c1d69[0m | |
| 2026-01-23T23:01:27.127+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35m1dbf7c04e9394591ad2d1e9d747c1d69[0m | |
| 2026-01-23T23:01:27.133+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:01:27.133+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:01:27.139+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m1dbf7c04e9394591ad2d1e9d747c1d69[0m | |
| 2026-01-23T23:01:27.139+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35m1dbf7c04e9394591ad2d1e9d747c1d69[0m | |
| 2026-01-23T23:01:27.139+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 0, 57, 139511, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m1dbf7c04e9394591ad2d1e9d747c1d69[0m | |
| 2026-01-23T23:01:27.139+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m1dbf7c04e9394591ad2d1e9d747c1d69[0m | |
| 2026-01-23T23:01:27.140+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35m1dbf7c04e9394591ad2d1e9d747c1d69[0m | |
| 2026-01-23T23:01:27.140+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m1dbf7c04e9394591ad2d1e9d747c1d69[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:01:27.140+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:41318 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:01:32.143+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m2a6fb677b74a486cbb4f6b7852783389[0m | |
| 2026-01-23T23:01:32.143+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35m2a6fb677b74a486cbb4f6b7852783389[0m | |
| 2026-01-23T23:01:32.149+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:01:32.149+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:01:32.158+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m2a6fb677b74a486cbb4f6b7852783389[0m | |
| 2026-01-23T23:01:32.159+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35m2a6fb677b74a486cbb4f6b7852783389[0m | |
| 2026-01-23T23:01:32.169+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:01:32.169+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:01:32.177+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m2a6fb677b74a486cbb4f6b7852783389[0m | |
| 2026-01-23T23:01:32.177+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35m2a6fb677b74a486cbb4f6b7852783389[0m | |
| 2026-01-23T23:01:32.178+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 1, 2, 177901, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m2a6fb677b74a486cbb4f6b7852783389[0m | |
| 2026-01-23T23:01:32.178+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m2a6fb677b74a486cbb4f6b7852783389[0m | |
| 2026-01-23T23:01:32.178+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35m2a6fb677b74a486cbb4f6b7852783389[0m | |
| 2026-01-23T23:01:32.178+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m2a6fb677b74a486cbb4f6b7852783389[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:01:32.179+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:34250 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:01:37.181+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35mced3d9773ae94fa0bf9cc376e05c4013[0m | |
| 2026-01-23T23:01:37.181+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35mced3d9773ae94fa0bf9cc376e05c4013[0m | |
| 2026-01-23T23:01:37.186+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:01:37.186+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:01:37.194+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35mced3d9773ae94fa0bf9cc376e05c4013[0m | |
| 2026-01-23T23:01:37.195+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35mced3d9773ae94fa0bf9cc376e05c4013[0m | |
| 2026-01-23T23:01:37.204+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:01:37.204+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:01:37.211+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35mced3d9773ae94fa0bf9cc376e05c4013[0m | |
| 2026-01-23T23:01:37.211+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35mced3d9773ae94fa0bf9cc376e05c4013[0m | |
| 2026-01-23T23:01:37.211+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 1, 7, 211235, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35mced3d9773ae94fa0bf9cc376e05c4013[0m | |
| 2026-01-23T23:01:37.211+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35mced3d9773ae94fa0bf9cc376e05c4013[0m | |
| 2026-01-23T23:01:37.211+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35mced3d9773ae94fa0bf9cc376e05c4013[0m | |
| 2026-01-23T23:01:37.211+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35mced3d9773ae94fa0bf9cc376e05c4013[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:01:37.212+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:34254 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:01:42.215+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35me8f1ec8483b642bda325e5552ec41d98[0m | |
| 2026-01-23T23:01:42.215+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35me8f1ec8483b642bda325e5552ec41d98[0m | |
| 2026-01-23T23:01:42.220+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:01:42.220+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:01:42.227+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35me8f1ec8483b642bda325e5552ec41d98[0m | |
| 2026-01-23T23:01:42.227+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35me8f1ec8483b642bda325e5552ec41d98[0m | |
| 2026-01-23T23:01:42.234+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:01:42.234+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:01:42.239+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35me8f1ec8483b642bda325e5552ec41d98[0m | |
| 2026-01-23T23:01:42.239+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35me8f1ec8483b642bda325e5552ec41d98[0m | |
| 2026-01-23T23:01:42.239+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 1, 12, 239898, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35me8f1ec8483b642bda325e5552ec41d98[0m | |
| 2026-01-23T23:01:42.240+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35me8f1ec8483b642bda325e5552ec41d98[0m | |
| 2026-01-23T23:01:42.240+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35me8f1ec8483b642bda325e5552ec41d98[0m | |
| 2026-01-23T23:01:42.240+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35me8f1ec8483b642bda325e5552ec41d98[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:01:42.240+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:39378 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:01:47.242+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m636f9b5e46d84812a1ce710078168da4[0m | |
| 2026-01-23T23:01:47.242+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35m636f9b5e46d84812a1ce710078168da4[0m | |
| 2026-01-23T23:01:47.246+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:01:47.246+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:01:47.252+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m636f9b5e46d84812a1ce710078168da4[0m | |
| 2026-01-23T23:01:47.252+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35m636f9b5e46d84812a1ce710078168da4[0m | |
| 2026-01-23T23:01:47.258+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:01:47.258+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:01:47.263+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m636f9b5e46d84812a1ce710078168da4[0m | |
| 2026-01-23T23:01:47.263+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35m636f9b5e46d84812a1ce710078168da4[0m | |
| 2026-01-23T23:01:47.263+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 1, 17, 263924, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m636f9b5e46d84812a1ce710078168da4[0m | |
| 2026-01-23T23:01:47.264+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m636f9b5e46d84812a1ce710078168da4[0m | |
| 2026-01-23T23:01:47.264+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35m636f9b5e46d84812a1ce710078168da4[0m | |
| 2026-01-23T23:01:47.264+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m636f9b5e46d84812a1ce710078168da4[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:01:47.265+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:39384 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:01:52.270+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m03a122db47934e49b191663ff1379ad9[0m | |
| 2026-01-23T23:01:52.271+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35m03a122db47934e49b191663ff1379ad9[0m | |
| 2026-01-23T23:01:52.278+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:01:52.279+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:01:52.289+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m03a122db47934e49b191663ff1379ad9[0m | |
| 2026-01-23T23:01:52.289+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35m03a122db47934e49b191663ff1379ad9[0m | |
| 2026-01-23T23:01:52.297+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:01:52.297+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:01:52.304+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m03a122db47934e49b191663ff1379ad9[0m | |
| 2026-01-23T23:01:52.304+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35m03a122db47934e49b191663ff1379ad9[0m | |
| 2026-01-23T23:01:52.305+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 1, 22, 305126, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m03a122db47934e49b191663ff1379ad9[0m | |
| 2026-01-23T23:01:52.305+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m03a122db47934e49b191663ff1379ad9[0m | |
| 2026-01-23T23:01:52.305+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35m03a122db47934e49b191663ff1379ad9[0m | |
| 2026-01-23T23:01:52.306+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m03a122db47934e49b191663ff1379ad9[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:01:52.306+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:54760 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:01:57.307+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m1ad7975ff6ff43f886e0d85e7d586eb8[0m | |
| 2026-01-23T23:01:57.308+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35m1ad7975ff6ff43f886e0d85e7d586eb8[0m | |
| 2026-01-23T23:01:57.313+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:01:57.314+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:01:57.322+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m1ad7975ff6ff43f886e0d85e7d586eb8[0m | |
| 2026-01-23T23:01:57.322+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35m1ad7975ff6ff43f886e0d85e7d586eb8[0m | |
| 2026-01-23T23:01:57.333+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:01:57.333+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:01:57.341+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m1ad7975ff6ff43f886e0d85e7d586eb8[0m | |
| 2026-01-23T23:01:57.341+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35m1ad7975ff6ff43f886e0d85e7d586eb8[0m | |
| 2026-01-23T23:01:57.341+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 1, 27, 341784, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m1ad7975ff6ff43f886e0d85e7d586eb8[0m | |
| 2026-01-23T23:01:57.341+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m1ad7975ff6ff43f886e0d85e7d586eb8[0m | |
| 2026-01-23T23:01:57.342+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35m1ad7975ff6ff43f886e0d85e7d586eb8[0m | |
| 2026-01-23T23:01:57.342+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m1ad7975ff6ff43f886e0d85e7d586eb8[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:01:57.342+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:54774 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:02:02.344+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35me913b964940b462d8b32c7297bf6b8ec[0m | |
| 2026-01-23T23:02:02.344+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35me913b964940b462d8b32c7297bf6b8ec[0m | |
| 2026-01-23T23:02:02.349+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:02:02.349+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:02:02.355+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35me913b964940b462d8b32c7297bf6b8ec[0m | |
| 2026-01-23T23:02:02.355+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35me913b964940b462d8b32c7297bf6b8ec[0m | |
| 2026-01-23T23:02:02.361+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:02:02.361+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:02:02.367+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35me913b964940b462d8b32c7297bf6b8ec[0m | |
| 2026-01-23T23:02:02.367+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35me913b964940b462d8b32c7297bf6b8ec[0m | |
| 2026-01-23T23:02:02.367+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 1, 32, 367936, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35me913b964940b462d8b32c7297bf6b8ec[0m | |
| 2026-01-23T23:02:02.368+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35me913b964940b462d8b32c7297bf6b8ec[0m | |
| 2026-01-23T23:02:02.368+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35me913b964940b462d8b32c7297bf6b8ec[0m | |
| 2026-01-23T23:02:02.368+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35me913b964940b462d8b32c7297bf6b8ec[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:02:02.368+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:38424 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:02:07.371+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35md7b5b1d13c08460381ad3af96263ef92[0m | |
| 2026-01-23T23:02:07.371+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35md7b5b1d13c08460381ad3af96263ef92[0m | |
| 2026-01-23T23:02:07.375+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:02:07.375+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:02:07.381+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35md7b5b1d13c08460381ad3af96263ef92[0m | |
| 2026-01-23T23:02:07.382+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35md7b5b1d13c08460381ad3af96263ef92[0m | |
| 2026-01-23T23:02:07.388+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:02:07.388+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:02:07.393+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35md7b5b1d13c08460381ad3af96263ef92[0m | |
| 2026-01-23T23:02:07.394+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35md7b5b1d13c08460381ad3af96263ef92[0m | |
| 2026-01-23T23:02:07.394+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 1, 37, 394329, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35md7b5b1d13c08460381ad3af96263ef92[0m | |
| 2026-01-23T23:02:07.394+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35md7b5b1d13c08460381ad3af96263ef92[0m | |
| 2026-01-23T23:02:07.395+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35md7b5b1d13c08460381ad3af96263ef92[0m | |
| 2026-01-23T23:02:07.395+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35md7b5b1d13c08460381ad3af96263ef92[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:02:07.395+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:38436 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:02:12.397+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m518f75cffd6a4969b3a57c5364cdb157[0m | |
| 2026-01-23T23:02:12.397+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35m518f75cffd6a4969b3a57c5364cdb157[0m | |
| 2026-01-23T23:02:12.401+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:02:12.401+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:02:12.407+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m518f75cffd6a4969b3a57c5364cdb157[0m | |
| 2026-01-23T23:02:12.408+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35m518f75cffd6a4969b3a57c5364cdb157[0m | |
| 2026-01-23T23:02:12.414+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:02:12.414+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:02:12.420+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m518f75cffd6a4969b3a57c5364cdb157[0m | |
| 2026-01-23T23:02:12.420+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35m518f75cffd6a4969b3a57c5364cdb157[0m | |
| 2026-01-23T23:02:12.420+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 1, 42, 420216, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m518f75cffd6a4969b3a57c5364cdb157[0m | |
| 2026-01-23T23:02:12.420+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m518f75cffd6a4969b3a57c5364cdb157[0m | |
| 2026-01-23T23:02:12.420+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35m518f75cffd6a4969b3a57c5364cdb157[0m | |
| 2026-01-23T23:02:12.420+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m518f75cffd6a4969b3a57c5364cdb157[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:02:12.420+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:59178 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:02:16.863+00:00 [33] [139636297664192] DEBUG PaisMTlsProviderWatcher probes/_report_provider_health - [1mReporting liveness: observer alive and provider healthy[0m | |
| 2026-01-23T23:02:16.863+00:00 [33] [139636297664192] DEBUG NamedCheckinHealthProbe probes/check_in - [1mReceived check-in [0m [36mname[0m=[35mmTLS-watcher[0m [36mnow[0m=[35mdatetime.datetime(2026, 1, 23, 23, 2, 16, 863658, tzinfo=datetime.timezone.utc)[0m | |
| 2026-01-23T23:02:17.421+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m862c57bf55f44432b459fc761588d4e7[0m | |
| 2026-01-23T23:02:17.422+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35m862c57bf55f44432b459fc761588d4e7[0m | |
| 2026-01-23T23:02:17.426+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:02:17.426+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:02:17.433+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m862c57bf55f44432b459fc761588d4e7[0m | |
| 2026-01-23T23:02:17.433+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35m862c57bf55f44432b459fc761588d4e7[0m | |
| 2026-01-23T23:02:17.439+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:02:17.439+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:02:17.445+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m862c57bf55f44432b459fc761588d4e7[0m | |
| 2026-01-23T23:02:17.445+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35m862c57bf55f44432b459fc761588d4e7[0m | |
| 2026-01-23T23:02:17.445+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 1, 47, 445660, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m862c57bf55f44432b459fc761588d4e7[0m | |
| 2026-01-23T23:02:17.445+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m862c57bf55f44432b459fc761588d4e7[0m | |
| 2026-01-23T23:02:17.445+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35m862c57bf55f44432b459fc761588d4e7[0m | |
| 2026-01-23T23:02:17.446+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m862c57bf55f44432b459fc761588d4e7[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:02:17.446+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:59182 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:02:18.483+00:00 [33] [139636530932608] DEBUG AsyncCachedCredentialsRepositoryWatcher _credentials/run - [1mReporting liveness: no events received[0m | |
| 2026-01-23T23:02:18.483+00:00 [33] [139636530932608] DEBUG NamedCheckinHealthProbe probes/check_in - [1mReceived check-in [0m [36mname[0m=[35minference-gateway-secret-watcher[0m [36mnow[0m=[35mdatetime.datetime(2026, 1, 23, 23, 2, 18, 483685, tzinfo=datetime.timezone.utc)[0m | |
| 2026-01-23T23:02:18.560+00:00 [33] [139636530932608] DEBUG AsyncCachedCredentialsRepositoryWatcher _credentials/run - [1mProcessing event [0m [36mevent_type[0m=[35mADDED[0m [36mresource_name[0m=[35mllm-service-token[0m | |
| 2026-01-23T23:02:18.560+00:00 [33] [139636530932608] DEBUG NamedCheckinHealthProbe probes/check_in - [1mReceived check-in [0m [36mname[0m=[35minference-gateway-secret-watcher[0m [36mnow[0m=[35mdatetime.datetime(2026, 1, 23, 23, 2, 18, 560718, tzinfo=datetime.timezone.utc)[0m | |
| 2026-01-23T23:02:22.447+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m256aed2350c54c7c8c8d50c6908d3de2[0m | |
| 2026-01-23T23:02:22.447+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35m256aed2350c54c7c8c8d50c6908d3de2[0m | |
| 2026-01-23T23:02:22.451+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:02:22.452+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:02:22.459+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m256aed2350c54c7c8c8d50c6908d3de2[0m | |
| 2026-01-23T23:02:22.459+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35m256aed2350c54c7c8c8d50c6908d3de2[0m | |
| 2026-01-23T23:02:22.465+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:02:22.465+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:02:22.470+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m256aed2350c54c7c8c8d50c6908d3de2[0m | |
| 2026-01-23T23:02:22.470+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35m256aed2350c54c7c8c8d50c6908d3de2[0m | |
| 2026-01-23T23:02:22.470+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 1, 52, 470615, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m256aed2350c54c7c8c8d50c6908d3de2[0m | |
| 2026-01-23T23:02:22.470+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m256aed2350c54c7c8c8d50c6908d3de2[0m | |
| 2026-01-23T23:02:22.470+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35m256aed2350c54c7c8c8d50c6908d3de2[0m | |
| 2026-01-23T23:02:22.471+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m256aed2350c54c7c8c8d50c6908d3de2[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:02:22.471+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:49936 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:02:27.474+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m445d47884b804800bd27ac06b6e1cf95[0m | |
| 2026-01-23T23:02:27.474+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35m445d47884b804800bd27ac06b6e1cf95[0m | |
| 2026-01-23T23:02:27.479+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:02:27.479+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:02:27.486+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m445d47884b804800bd27ac06b6e1cf95[0m | |
| 2026-01-23T23:02:27.487+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35m445d47884b804800bd27ac06b6e1cf95[0m | |
| 2026-01-23T23:02:27.493+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:02:27.493+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:02:27.498+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m445d47884b804800bd27ac06b6e1cf95[0m | |
| 2026-01-23T23:02:27.499+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35m445d47884b804800bd27ac06b6e1cf95[0m | |
| 2026-01-23T23:02:27.499+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 1, 57, 499127, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m445d47884b804800bd27ac06b6e1cf95[0m | |
| 2026-01-23T23:02:27.499+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m445d47884b804800bd27ac06b6e1cf95[0m | |
| 2026-01-23T23:02:27.499+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35m445d47884b804800bd27ac06b6e1cf95[0m | |
| 2026-01-23T23:02:27.499+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m445d47884b804800bd27ac06b6e1cf95[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:02:27.499+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:49940 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:02:32.501+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m9ae9efa7e8d3451aa1a651883d9d0357[0m | |
| 2026-01-23T23:02:32.502+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35m9ae9efa7e8d3451aa1a651883d9d0357[0m | |
| 2026-01-23T23:02:32.507+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:02:32.508+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:02:32.515+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m9ae9efa7e8d3451aa1a651883d9d0357[0m | |
| 2026-01-23T23:02:32.515+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35m9ae9efa7e8d3451aa1a651883d9d0357[0m | |
| 2026-01-23T23:02:32.525+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:02:32.525+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:02:32.533+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m9ae9efa7e8d3451aa1a651883d9d0357[0m | |
| 2026-01-23T23:02:32.533+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35m9ae9efa7e8d3451aa1a651883d9d0357[0m | |
| 2026-01-23T23:02:32.534+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 2, 2, 534234, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m9ae9efa7e8d3451aa1a651883d9d0357[0m | |
| 2026-01-23T23:02:32.534+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m9ae9efa7e8d3451aa1a651883d9d0357[0m | |
| 2026-01-23T23:02:32.534+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35m9ae9efa7e8d3451aa1a651883d9d0357[0m | |
| 2026-01-23T23:02:32.534+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m9ae9efa7e8d3451aa1a651883d9d0357[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:02:32.534+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:39720 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:02:37.540+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m68aaea13cd974ecc8b8f339d2b5ecbd3[0m | |
| 2026-01-23T23:02:37.541+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35m68aaea13cd974ecc8b8f339d2b5ecbd3[0m | |
| 2026-01-23T23:02:37.548+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:02:37.548+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:02:37.557+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m68aaea13cd974ecc8b8f339d2b5ecbd3[0m | |
| 2026-01-23T23:02:37.558+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35m68aaea13cd974ecc8b8f339d2b5ecbd3[0m | |
| 2026-01-23T23:02:37.564+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:02:37.564+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:02:37.570+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m68aaea13cd974ecc8b8f339d2b5ecbd3[0m | |
| 2026-01-23T23:02:37.570+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35m68aaea13cd974ecc8b8f339d2b5ecbd3[0m | |
| 2026-01-23T23:02:37.570+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 2, 7, 570676, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m68aaea13cd974ecc8b8f339d2b5ecbd3[0m | |
| 2026-01-23T23:02:37.570+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m68aaea13cd974ecc8b8f339d2b5ecbd3[0m | |
| 2026-01-23T23:02:37.570+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35m68aaea13cd974ecc8b8f339d2b5ecbd3[0m | |
| 2026-01-23T23:02:37.571+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m68aaea13cd974ecc8b8f339d2b5ecbd3[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:02:37.571+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:39734 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:02:42.573+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35mb0e9aefff3bc41119c108de8d55f3c75[0m | |
| 2026-01-23T23:02:42.573+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35mb0e9aefff3bc41119c108de8d55f3c75[0m | |
| 2026-01-23T23:02:42.577+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:02:42.577+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:02:42.583+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35mb0e9aefff3bc41119c108de8d55f3c75[0m | |
| 2026-01-23T23:02:42.584+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35mb0e9aefff3bc41119c108de8d55f3c75[0m | |
| 2026-01-23T23:02:42.590+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:02:42.590+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:02:42.596+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35mb0e9aefff3bc41119c108de8d55f3c75[0m | |
| 2026-01-23T23:02:42.596+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35mb0e9aefff3bc41119c108de8d55f3c75[0m | |
| 2026-01-23T23:02:42.596+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 2, 12, 596635, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35mb0e9aefff3bc41119c108de8d55f3c75[0m | |
| 2026-01-23T23:02:42.596+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35mb0e9aefff3bc41119c108de8d55f3c75[0m | |
| 2026-01-23T23:02:42.596+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35mb0e9aefff3bc41119c108de8d55f3c75[0m | |
| 2026-01-23T23:02:42.597+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35mb0e9aefff3bc41119c108de8d55f3c75[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:02:42.597+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:33272 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:02:47.598+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35me5e1481edf6d455db76d95efbc4806e7[0m | |
| 2026-01-23T23:02:47.598+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35me5e1481edf6d455db76d95efbc4806e7[0m | |
| 2026-01-23T23:02:47.602+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:02:47.603+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:02:47.610+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35me5e1481edf6d455db76d95efbc4806e7[0m | |
| 2026-01-23T23:02:47.610+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35me5e1481edf6d455db76d95efbc4806e7[0m | |
| 2026-01-23T23:02:47.616+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:02:47.616+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:02:47.622+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35me5e1481edf6d455db76d95efbc4806e7[0m | |
| 2026-01-23T23:02:47.622+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35me5e1481edf6d455db76d95efbc4806e7[0m | |
| 2026-01-23T23:02:47.622+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 2, 17, 622411, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35me5e1481edf6d455db76d95efbc4806e7[0m | |
| 2026-01-23T23:02:47.622+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35me5e1481edf6d455db76d95efbc4806e7[0m | |
| 2026-01-23T23:02:47.622+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35me5e1481edf6d455db76d95efbc4806e7[0m | |
| 2026-01-23T23:02:47.622+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35me5e1481edf6d455db76d95efbc4806e7[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:02:47.622+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:33288 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:02:52.628+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m17b5ee75d28945858b6352ecdc531d71[0m | |
| 2026-01-23T23:02:52.629+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35m17b5ee75d28945858b6352ecdc531d71[0m | |
| 2026-01-23T23:02:52.636+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:02:52.636+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:02:52.645+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m17b5ee75d28945858b6352ecdc531d71[0m | |
| 2026-01-23T23:02:52.645+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35m17b5ee75d28945858b6352ecdc531d71[0m | |
| 2026-01-23T23:02:52.652+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:02:52.652+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:02:52.657+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m17b5ee75d28945858b6352ecdc531d71[0m | |
| 2026-01-23T23:02:52.657+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35m17b5ee75d28945858b6352ecdc531d71[0m | |
| 2026-01-23T23:02:52.658+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 2, 22, 658152, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m17b5ee75d28945858b6352ecdc531d71[0m | |
| 2026-01-23T23:02:52.658+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m17b5ee75d28945858b6352ecdc531d71[0m | |
| 2026-01-23T23:02:52.658+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35m17b5ee75d28945858b6352ecdc531d71[0m | |
| 2026-01-23T23:02:52.658+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m17b5ee75d28945858b6352ecdc531d71[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:02:52.659+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:48372 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:02:57.661+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m34214fa7657e44a79ef29489d42ece7c[0m | |
| 2026-01-23T23:02:57.661+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35m34214fa7657e44a79ef29489d42ece7c[0m | |
| 2026-01-23T23:02:57.666+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:02:57.666+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:02:57.673+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m34214fa7657e44a79ef29489d42ece7c[0m | |
| 2026-01-23T23:02:57.674+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35m34214fa7657e44a79ef29489d42ece7c[0m | |
| 2026-01-23T23:02:57.680+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:02:57.680+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:02:57.686+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m34214fa7657e44a79ef29489d42ece7c[0m | |
| 2026-01-23T23:02:57.686+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35m34214fa7657e44a79ef29489d42ece7c[0m | |
| 2026-01-23T23:02:57.686+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 2, 27, 686692, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m34214fa7657e44a79ef29489d42ece7c[0m | |
| 2026-01-23T23:02:57.686+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m34214fa7657e44a79ef29489d42ece7c[0m | |
| 2026-01-23T23:02:57.686+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35m34214fa7657e44a79ef29489d42ece7c[0m | |
| 2026-01-23T23:02:57.687+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m34214fa7657e44a79ef29489d42ece7c[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:02:57.687+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:48388 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:03:02.688+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35ma2c0c090cfa84d88ac781b67b946ace2[0m | |
| 2026-01-23T23:03:02.689+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35ma2c0c090cfa84d88ac781b67b946ace2[0m | |
| 2026-01-23T23:03:02.694+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:03:02.694+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:03:02.701+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35ma2c0c090cfa84d88ac781b67b946ace2[0m | |
| 2026-01-23T23:03:02.701+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35ma2c0c090cfa84d88ac781b67b946ace2[0m | |
| 2026-01-23T23:03:02.708+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:03:02.708+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:03:02.713+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35ma2c0c090cfa84d88ac781b67b946ace2[0m | |
| 2026-01-23T23:03:02.713+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35ma2c0c090cfa84d88ac781b67b946ace2[0m | |
| 2026-01-23T23:03:02.713+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 2, 32, 713605, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35ma2c0c090cfa84d88ac781b67b946ace2[0m | |
| 2026-01-23T23:03:02.713+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35ma2c0c090cfa84d88ac781b67b946ace2[0m | |
| 2026-01-23T23:03:02.713+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35ma2c0c090cfa84d88ac781b67b946ace2[0m | |
| 2026-01-23T23:03:02.714+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35ma2c0c090cfa84d88ac781b67b946ace2[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:03:02.714+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:53850 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:03:07.715+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m2ba12ddb32384b89825be61381c9beb3[0m | |
| 2026-01-23T23:03:07.715+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35m2ba12ddb32384b89825be61381c9beb3[0m | |
| 2026-01-23T23:03:07.720+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:03:07.720+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:03:07.727+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m2ba12ddb32384b89825be61381c9beb3[0m | |
| 2026-01-23T23:03:07.727+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35m2ba12ddb32384b89825be61381c9beb3[0m | |
| 2026-01-23T23:03:07.733+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:03:07.733+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:03:07.740+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m2ba12ddb32384b89825be61381c9beb3[0m | |
| 2026-01-23T23:03:07.740+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35m2ba12ddb32384b89825be61381c9beb3[0m | |
| 2026-01-23T23:03:07.740+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 2, 37, 740234, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m2ba12ddb32384b89825be61381c9beb3[0m | |
| 2026-01-23T23:03:07.740+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m2ba12ddb32384b89825be61381c9beb3[0m | |
| 2026-01-23T23:03:07.740+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35m2ba12ddb32384b89825be61381c9beb3[0m | |
| 2026-01-23T23:03:07.740+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m2ba12ddb32384b89825be61381c9beb3[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:03:07.740+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:53852 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:03:12.742+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m13eb68f5611f44b79a70fd17a0ebbb36[0m | |
| 2026-01-23T23:03:12.742+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35m13eb68f5611f44b79a70fd17a0ebbb36[0m | |
| 2026-01-23T23:03:12.746+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:03:12.746+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:03:12.753+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m13eb68f5611f44b79a70fd17a0ebbb36[0m | |
| 2026-01-23T23:03:12.753+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35m13eb68f5611f44b79a70fd17a0ebbb36[0m | |
| 2026-01-23T23:03:12.761+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:03:12.761+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:03:12.767+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m13eb68f5611f44b79a70fd17a0ebbb36[0m | |
| 2026-01-23T23:03:12.767+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35m13eb68f5611f44b79a70fd17a0ebbb36[0m | |
| 2026-01-23T23:03:12.768+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 2, 42, 767995, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m13eb68f5611f44b79a70fd17a0ebbb36[0m | |
| 2026-01-23T23:03:12.768+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m13eb68f5611f44b79a70fd17a0ebbb36[0m | |
| 2026-01-23T23:03:12.768+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35m13eb68f5611f44b79a70fd17a0ebbb36[0m | |
| 2026-01-23T23:03:12.768+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m13eb68f5611f44b79a70fd17a0ebbb36[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:03:12.768+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:59346 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:03:16.864+00:00 [33] [139636297664192] DEBUG PaisMTlsProviderWatcher probes/_report_provider_health - [1mReporting liveness: observer alive and provider healthy[0m | |
| 2026-01-23T23:03:16.864+00:00 [33] [139636297664192] DEBUG NamedCheckinHealthProbe probes/check_in - [1mReceived check-in [0m [36mname[0m=[35mmTLS-watcher[0m [36mnow[0m=[35mdatetime.datetime(2026, 1, 23, 23, 3, 16, 864874, tzinfo=datetime.timezone.utc)[0m | |
| 2026-01-23T23:03:17.770+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m651a9effbd4b4deda5d297c6c287f19f[0m | |
| 2026-01-23T23:03:17.770+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35m651a9effbd4b4deda5d297c6c287f19f[0m | |
| 2026-01-23T23:03:17.774+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:03:17.774+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:03:17.780+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m651a9effbd4b4deda5d297c6c287f19f[0m | |
| 2026-01-23T23:03:17.780+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35m651a9effbd4b4deda5d297c6c287f19f[0m | |
| 2026-01-23T23:03:17.786+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:03:17.787+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:03:17.793+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m651a9effbd4b4deda5d297c6c287f19f[0m | |
| 2026-01-23T23:03:17.793+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35m651a9effbd4b4deda5d297c6c287f19f[0m | |
| 2026-01-23T23:03:17.793+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 2, 47, 793659, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m651a9effbd4b4deda5d297c6c287f19f[0m | |
| 2026-01-23T23:03:17.793+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m651a9effbd4b4deda5d297c6c287f19f[0m | |
| 2026-01-23T23:03:17.794+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35m651a9effbd4b4deda5d297c6c287f19f[0m | |
| 2026-01-23T23:03:17.794+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m651a9effbd4b4deda5d297c6c287f19f[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:03:17.794+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:59358 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:03:18.560+00:00 [33] [139636530932608] DEBUG AsyncCachedCredentialsRepositoryWatcher _credentials/run - [1mReporting liveness: no events received[0m | |
| 2026-01-23T23:03:18.561+00:00 [33] [139636530932608] DEBUG NamedCheckinHealthProbe probes/check_in - [1mReceived check-in [0m [36mname[0m=[35minference-gateway-secret-watcher[0m [36mnow[0m=[35mdatetime.datetime(2026, 1, 23, 23, 3, 18, 561152, tzinfo=datetime.timezone.utc)[0m | |
| 2026-01-23T23:03:18.636+00:00 [33] [139636530932608] DEBUG AsyncCachedCredentialsRepositoryWatcher _credentials/run - [1mProcessing event [0m [36mevent_type[0m=[35mADDED[0m [36mresource_name[0m=[35mllm-service-token[0m | |
| 2026-01-23T23:03:18.636+00:00 [33] [139636530932608] DEBUG NamedCheckinHealthProbe probes/check_in - [1mReceived check-in [0m [36mname[0m=[35minference-gateway-secret-watcher[0m [36mnow[0m=[35mdatetime.datetime(2026, 1, 23, 23, 3, 18, 636868, tzinfo=datetime.timezone.utc)[0m | |
| 2026-01-23T23:03:22.796+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35mb12f4fc7f0264e79abda891000b6b230[0m | |
| 2026-01-23T23:03:22.796+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35mb12f4fc7f0264e79abda891000b6b230[0m | |
| 2026-01-23T23:03:22.800+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:03:22.800+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:03:22.807+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35mb12f4fc7f0264e79abda891000b6b230[0m | |
| 2026-01-23T23:03:22.807+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35mb12f4fc7f0264e79abda891000b6b230[0m | |
| 2026-01-23T23:03:22.813+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:03:22.813+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:03:22.820+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35mb12f4fc7f0264e79abda891000b6b230[0m | |
| 2026-01-23T23:03:22.820+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35mb12f4fc7f0264e79abda891000b6b230[0m | |
| 2026-01-23T23:03:22.820+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 2, 52, 820437, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35mb12f4fc7f0264e79abda891000b6b230[0m | |
| 2026-01-23T23:03:22.820+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35mb12f4fc7f0264e79abda891000b6b230[0m | |
| 2026-01-23T23:03:22.820+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35mb12f4fc7f0264e79abda891000b6b230[0m | |
| 2026-01-23T23:03:22.821+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35mb12f4fc7f0264e79abda891000b6b230[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:03:22.821+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:33124 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:03:27.822+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m60f7ce8f8a7c48acb81fb3581893135d[0m | |
| 2026-01-23T23:03:27.823+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35m60f7ce8f8a7c48acb81fb3581893135d[0m | |
| 2026-01-23T23:03:27.827+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:03:27.827+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:03:27.836+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m60f7ce8f8a7c48acb81fb3581893135d[0m | |
| 2026-01-23T23:03:27.836+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35m60f7ce8f8a7c48acb81fb3581893135d[0m | |
| 2026-01-23T23:03:27.841+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:03:27.841+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:03:27.846+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m60f7ce8f8a7c48acb81fb3581893135d[0m | |
| 2026-01-23T23:03:27.847+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35m60f7ce8f8a7c48acb81fb3581893135d[0m | |
| 2026-01-23T23:03:27.847+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 2, 57, 847247, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m60f7ce8f8a7c48acb81fb3581893135d[0m | |
| 2026-01-23T23:03:27.847+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m60f7ce8f8a7c48acb81fb3581893135d[0m | |
| 2026-01-23T23:03:27.847+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35m60f7ce8f8a7c48acb81fb3581893135d[0m | |
| 2026-01-23T23:03:27.847+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m60f7ce8f8a7c48acb81fb3581893135d[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:03:27.848+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:33126 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:03:32.849+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35mcead12baa3464daab450f5634d759d51[0m | |
| 2026-01-23T23:03:32.849+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35mcead12baa3464daab450f5634d759d51[0m | |
| 2026-01-23T23:03:32.853+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:03:32.853+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:03:32.859+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35mcead12baa3464daab450f5634d759d51[0m | |
| 2026-01-23T23:03:32.859+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35mcead12baa3464daab450f5634d759d51[0m | |
| 2026-01-23T23:03:32.865+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:03:32.865+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:03:32.870+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35mcead12baa3464daab450f5634d759d51[0m | |
| 2026-01-23T23:03:32.870+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35mcead12baa3464daab450f5634d759d51[0m | |
| 2026-01-23T23:03:32.870+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 3, 2, 870875, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35mcead12baa3464daab450f5634d759d51[0m | |
| 2026-01-23T23:03:32.871+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35mcead12baa3464daab450f5634d759d51[0m | |
| 2026-01-23T23:03:32.871+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35mcead12baa3464daab450f5634d759d51[0m | |
| 2026-01-23T23:03:32.871+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35mcead12baa3464daab450f5634d759d51[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:03:32.871+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:54478 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:03:37.873+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35mba005927b3ed4e8db355123bcb3885b6[0m | |
| 2026-01-23T23:03:37.873+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35mba005927b3ed4e8db355123bcb3885b6[0m | |
| 2026-01-23T23:03:37.877+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:03:37.877+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:03:37.883+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35mba005927b3ed4e8db355123bcb3885b6[0m | |
| 2026-01-23T23:03:37.883+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35mba005927b3ed4e8db355123bcb3885b6[0m | |
| 2026-01-23T23:03:37.889+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:03:37.889+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:03:37.895+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35mba005927b3ed4e8db355123bcb3885b6[0m | |
| 2026-01-23T23:03:37.895+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35mba005927b3ed4e8db355123bcb3885b6[0m | |
| 2026-01-23T23:03:37.896+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 3, 7, 895964, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35mba005927b3ed4e8db355123bcb3885b6[0m | |
| 2026-01-23T23:03:37.896+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35mba005927b3ed4e8db355123bcb3885b6[0m | |
| 2026-01-23T23:03:37.896+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35mba005927b3ed4e8db355123bcb3885b6[0m | |
| 2026-01-23T23:03:37.896+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35mba005927b3ed4e8db355123bcb3885b6[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:03:37.896+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:54490 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:03:42.898+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m8b1ca9eeb93145639ed5295d81a9fc1f[0m | |
| 2026-01-23T23:03:42.898+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35m8b1ca9eeb93145639ed5295d81a9fc1f[0m | |
| 2026-01-23T23:03:42.902+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:03:42.902+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:03:42.908+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m8b1ca9eeb93145639ed5295d81a9fc1f[0m | |
| 2026-01-23T23:03:42.908+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35m8b1ca9eeb93145639ed5295d81a9fc1f[0m | |
| 2026-01-23T23:03:42.914+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:03:42.914+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:03:42.920+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m8b1ca9eeb93145639ed5295d81a9fc1f[0m | |
| 2026-01-23T23:03:42.920+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35m8b1ca9eeb93145639ed5295d81a9fc1f[0m | |
| 2026-01-23T23:03:42.921+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 3, 12, 921104, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m8b1ca9eeb93145639ed5295d81a9fc1f[0m | |
| 2026-01-23T23:03:42.921+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m8b1ca9eeb93145639ed5295d81a9fc1f[0m | |
| 2026-01-23T23:03:42.921+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35m8b1ca9eeb93145639ed5295d81a9fc1f[0m | |
| 2026-01-23T23:03:42.921+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m8b1ca9eeb93145639ed5295d81a9fc1f[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:03:42.922+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:36370 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:03:47.923+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35mb4b6c085332d4d38aa2c22332d003a34[0m | |
| 2026-01-23T23:03:47.923+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35mb4b6c085332d4d38aa2c22332d003a34[0m | |
| 2026-01-23T23:03:47.927+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:03:47.928+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:03:47.934+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35mb4b6c085332d4d38aa2c22332d003a34[0m | |
| 2026-01-23T23:03:47.934+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35mb4b6c085332d4d38aa2c22332d003a34[0m | |
| 2026-01-23T23:03:47.942+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:03:47.942+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:03:47.948+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35mb4b6c085332d4d38aa2c22332d003a34[0m | |
| 2026-01-23T23:03:47.948+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35mb4b6c085332d4d38aa2c22332d003a34[0m | |
| 2026-01-23T23:03:47.948+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 3, 17, 948444, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35mb4b6c085332d4d38aa2c22332d003a34[0m | |
| 2026-01-23T23:03:47.948+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35mb4b6c085332d4d38aa2c22332d003a34[0m | |
| 2026-01-23T23:03:47.948+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35mb4b6c085332d4d38aa2c22332d003a34[0m | |
| 2026-01-23T23:03:47.949+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35mb4b6c085332d4d38aa2c22332d003a34[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:03:47.949+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:36382 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:03:52.952+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m7c088df97a5a491baa5c5713defbe999[0m | |
| 2026-01-23T23:03:52.953+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35m7c088df97a5a491baa5c5713defbe999[0m | |
| 2026-01-23T23:03:52.960+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:03:52.960+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:03:52.970+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m7c088df97a5a491baa5c5713defbe999[0m | |
| 2026-01-23T23:03:52.970+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35m7c088df97a5a491baa5c5713defbe999[0m | |
| 2026-01-23T23:03:52.976+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:03:52.976+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:03:52.982+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m7c088df97a5a491baa5c5713defbe999[0m | |
| 2026-01-23T23:03:52.985+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35m7c088df97a5a491baa5c5713defbe999[0m | |
| 2026-01-23T23:03:52.985+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 3, 22, 985771, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m7c088df97a5a491baa5c5713defbe999[0m | |
| 2026-01-23T23:03:52.986+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m7c088df97a5a491baa5c5713defbe999[0m | |
| 2026-01-23T23:03:52.986+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35m7c088df97a5a491baa5c5713defbe999[0m | |
| 2026-01-23T23:03:52.987+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m7c088df97a5a491baa5c5713defbe999[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:03:52.987+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:34212 - "GET /internal/health?probe=startup HTTP/1.1" 500 | |
| 2026-01-23T23:03:57.989+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mIncoming API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m953691b36525478b9bc1435c00238480[0m | |
| 2026-01-23T23:03:57.989+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mEvaluating health [0m [36mnum_probes[0m=[35m5[0m [36mrequest_id[0m=[35m953691b36525478b9bc1435c00238480[0m | |
| 2026-01-23T23:03:57.994+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:03:57.994+00:00 [33] [139635905763008] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:03:58.001+00:00 [33] [139635905763008] INFO db_version version/assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m953691b36525478b9bc1435c00238480[0m | |
| 2026-01-23T23:03:58.001+00:00 [33] [139635905763008] DEBUG DBConnectivitySyncHealthProbe probes/_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-sync[0m [36mrequest_id[0m=[35m953691b36525478b9bc1435c00238480[0m | |
| 2026-01-23T23:03:58.010+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Context impl PostgresqlImpl. | |
| 2026-01-23T23:03:58.010+00:00 [33] [139636530932608] INFO alembic.runtime.migration migration/__init__ - Will assume transactional DDL. | |
| 2026-01-23T23:03:58.015+00:00 [33] [139636530932608] INFO db_version version/async_assert_versions_match - [1mChecking database version [0m [36mcurrent_rev[0m=[35m68608a65fcf7[0m [36mnewest_rev[0m=[35m68608a65fcf7[0m [36mrequest_id[0m=[35m953691b36525478b9bc1435c00238480[0m | |
| 2026-01-23T23:03:58.016+00:00 [33] [139636530932608] DEBUG DBConnectivityAsyncHealthProbe probes/_async_probe - [1mProbe succeeded [0m [36mname[0m=[35mdb-connectivity-async[0m [36mrequest_id[0m=[35m953691b36525478b9bc1435c00238480[0m | |
| 2026-01-23T23:03:58.016+00:00 [33] [139636530932608] DEBUG RouteDiscoveryHealthProbe probes/_probe - [1mReporting error [0m [36mcheck_in_deadline[0m=[35mdatetime.datetime(2026, 1, 23, 23, 3, 28, 16178, tzinfo=datetime.timezone.utc)[0m [36mlast_check_in[0m=[35mNone[0m [36mmax_age[0m=[35mdatetime.timedelta(seconds=30)[0m [36mname[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m953691b36525478b9bc1435c00238480[0m | |
| 2026-01-23T23:03:58.016+00:00 [33] [139636530932608] WARNING health _routes/handler - [1mProbe indicated not healthy [0m [36mdetails[0m=[35mDiscovery of Inference Gateway Routes has not yet started[0m [36mprobe_name[0m=[35minference-gateway-route-discovery[0m [36mrequest_id[0m=[35m953691b36525478b9bc1435c00238480[0m | |
| 2026-01-23T23:03:58.016+00:00 [33] [139636530932608] DEBUG health _routes/handler - [1mReporting health [0m [36mhealthy[0m=[35mFalse[0m [36mrequest_id[0m=[35m953691b36525478b9bc1435c00238480[0m | |
| 2026-01-23T23:03:58.016+00:00 [33] [139636530932608] INFO request_response_logger _middleware/dispatch - [1mFinished API request: [0m [36mmethod[0m=[35mGET[0m [36mpath[0m=[35m/internal/health[0m [36mrequest_id[0m=[35m953691b36525478b9bc1435c00238480[0m [36mstatus_code[0m=[35m<HTTPStatus.INTERNAL_SERVER_ERROR: 500>[0m | |
| 2026-01-23T23:03:58.016+00:00 [33] [139636530932608] INFO uvicorn.access httptools_impl/send - 10.160.7.138:34222 - "GET /internal/health?probe=startup HTTP/1.1" 500 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment