QueuePool limit of AQLAlchemy
At some point, SQLAlchemy has not enough connections to distribute to the running threads (I assume). A `TimeoutError) is raised, pointing at https://docs.sqlalchemy.org/en/13/errors.html#error-3o7r
I should inspect this and make sure no connection is left open when Threads finish. Also, while a thread is waiting, the connection should be closed.
agent_tos_1 | [2020-09-03 08:07:44 +0000] [14] [ERROR] Exception in ASGI application
agent_tos_1 | Traceback (most recent call last):
agent_tos_1 | File "/usr/local/lib/python3.8/site-packages/uvicorn/protocols/http/httptools_impl.py", line 386, in run_asgi
agent_tos_1 | result = await app(self.scope, self.receive, self.send)
agent_tos_1 | File "/usr/local/lib/python3.8/site-packages/uvicorn/middleware/proxy_headers.py", line 45, in __call__
agent_tos_1 | return await self.app(scope, receive, send)
agent_tos_1 | File "/usr/local/lib/python3.8/site-packages/fastapi/applications.py", line 181, in __call__
agent_tos_1 | await super().__call__(scope, receive, send)
agent_tos_1 | File "/usr/local/lib/python3.8/site-packages/starlette/applications.py", line 102, in __call__
agent_tos_1 | await self.middleware_stack(scope, receive, send)
agent_tos_1 | File "/usr/local/lib/python3.8/site-packages/starlette/middleware/errors.py", line 181, in __call__
agent_tos_1 | raise exc from None
agent_tos_1 | File "/usr/local/lib/python3.8/site-packages/starlette/middleware/errors.py", line 159, in __call__
agent_tos_1 | await self.app(scope, receive, _send)
agent_tos_1 | File "/usr/local/lib/python3.8/site-packages/starlette/exceptions.py", line 82, in __call__
agent_tos_1 | raise exc from None
agent_tos_1 | File "/usr/local/lib/python3.8/site-packages/starlette/exceptions.py", line 71, in __call__
agent_tos_1 | await self.app(scope, receive, sender)
agent_tos_1 | File "/usr/local/lib/python3.8/site-packages/starlette/routing.py", line 550, in __call__
agent_tos_1 | await route.handle(scope, receive, send)
agent_tos_1 | File "/usr/local/lib/python3.8/site-packages/starlette/routing.py", line 227, in handle
agent_tos_1 | await self.app(scope, receive, send)
agent_tos_1 | File "/usr/local/lib/python3.8/site-packages/starlette/routing.py", line 41, in app
agent_tos_1 | response = await func(request)
agent_tos_1 | File "/usr/local/lib/python3.8/site-packages/fastapi/routing.py", line 196, in app
agent_tos_1 | raw_response = await run_endpoint_function(
agent_tos_1 | File "/usr/local/lib/python3.8/site-packages/fastapi/routing.py", line 149, in run_endpoint_function
agent_tos_1 | return await run_in_threadpool(dependant.call, **values)
agent_tos_1 | File "/usr/local/lib/python3.8/site-packages/starlette/concurrency.py", line 34, in run_in_threadpool
agent_tos_1 | return await loop.run_in_executor(None, func, *args)
agent_tos_1 | File "/usr/local/lib/python3.8/concurrent/futures/thread.py", line 57, in run
agent_tos_1 | result = self.fn(*self.args, **self.kwargs)
agent_tos_1 | File "/app/tos_execute/main.py", line 58, in tos_executor
agent_tos_1 | return register(
agent_tos_1 | File "/app/tos_execute/main.py", line 93, in register
agent_tos_1 | jobs = store.get_jobs_by_availability(True, db)
agent_tos_1 | File "/app/tos_execute/store.py", line 21, in get_jobs_by_availability
agent_tos_1 | return db.query(intmodel.Job).filter(intmodel.Job.available == avail).all()
agent_tos_1 | File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 3341, in all
agent_tos_1 | return list(self)
agent_tos_1 | File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 3503, in __iter__
agent_tos_1 | return self._execute_and_instances(context)
agent_tos_1 | File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 3524, in _execute_and_instances
agent_tos_1 | conn = self._get_bind_args(
agent_tos_1 | File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 3539, in _get_bind_args
agent_tos_1 | return fn(
agent_tos_1 | File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 3518, in _connection_from_session
agent_tos_1 | conn = self.session.connection(**kw)
agent_tos_1 | File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 1138, in connection
agent_tos_1 | return self._connection_for_bind(
agent_tos_1 | File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 1146, in _connection_for_bind
agent_tos_1 | return self.transaction._connection_for_bind(
agent_tos_1 | File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 433, in _connection_for_bind
agent_tos_1 | conn = bind._contextual_connect()
agent_tos_1 | File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 2305, in _contextual_connect
agent_tos_1 | self._wrap_pool_connect(self.pool.connect, None),
agent_tos_1 | File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 2339, in _wrap_pool_connect
agent_tos_1 | return fn()
agent_tos_1 | File "/usr/local/lib/python3.8/site-packages/sqlalchemy/pool/base.py", line 364, in connect
agent_tos_1 | return _ConnectionFairy._checkout(self)
agent_tos_1 | File "/usr/local/lib/python3.8/site-packages/sqlalchemy/pool/base.py", line 778, in _checkout
agent_tos_1 | fairy = _ConnectionRecord.checkout(pool)
agent_tos_1 | File "/usr/local/lib/python3.8/site-packages/sqlalchemy/pool/base.py", line 495, in checkout
agent_tos_1 | rec = pool._do_get()
agent_tos_1 | File "/usr/local/lib/python3.8/site-packages/sqlalchemy/pool/impl.py", line 128, in _do_get
agent_tos_1 | raise exc.TimeoutError(
agent_tos_1 | sqlalchemy.exc.TimeoutError: QueuePool limit of size 5 overflow 10 reached, connection timed out, timeout 30 (Background on this error at: http://sqlalche.me/e/13/3o7r)