prefect: 2.0b11 throws sql alchemy migration error when deploying (409 conflict error after db upgrade)
I am using the python API to submit a Deployment
. The following migration error is thrown.
Traceback (most recent call last):
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1819, in _execute_context
self.dialect.do_execute(
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/sqlalchemy/engine/default.py", line 732, in do_execute
cursor.execute(statement, parameters)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/sqlalchemy/dialects/sqlite/aiosqlite.py", line 100, in execute
self._adapt_connection._handle_exception(error)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/sqlalchemy/dialects/sqlite/aiosqlite.py", line 228, in _handle_exception
raise error
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/sqlalchemy/dialects/sqlite/aiosqlite.py", line 82, in execute
self.await_(_cursor.execute(operation, parameters))
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 68, in await_only
return current.driver.switch(awaitable)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 121, in greenlet_spawn
value = await result
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/aiosqlite/cursor.py", line 37, in execute
await self._execute(self._cursor.execute, sql, parameters)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/aiosqlite/cursor.py", line 31, in _execute
return await self._conn._execute(fn, *args, **kwargs)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/aiosqlite/core.py", line 129, in _execute
return await future
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/aiosqlite/core.py", line 102, in run
result = function()
sqlite3.OperationalError: no such table: flow_run_alert_policy
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "prefect_submit.py", line 197, in <module>
app.run(run)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/absl/app.py", line 312, in run
_run_main(main, args)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/absl/app.py", line 258, in _run_main
sys.exit(main(argv))
File "prefect_submit.py", line 193, in run
asyncio.run(main(script_name, ecr_docker_image_uri))
File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
return future.result()
File "prefect_submit.py", line 180, in main
deployment_id = await spec.create(client=client)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/prefect/client.py", line 105, in with_injected_client
return await fn(*args, **kwargs)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/prefect/deployments.py", line 226, in create
infrastructure_document_id = await infrastructure._save(is_anonymous=True)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/prefect/blocks/core.py", line 555, in _save
async with prefect.client.get_client() as client:
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/prefect/client.py", line 1898, in __aenter__
self._ephemeral_lifespan = await self._exit_stack.enter_async_context(
File "/usr/lib/python3.8/contextlib.py", line 568, in enter_async_context
result = await _cm_type.__aenter__(cm)
File "/usr/lib/python3.8/contextlib.py", line 171, in __aenter__
return await self.gen.__anext__()
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/prefect/client.py", line 184, in app_lifespan_context
await context.__aenter__()
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/asgi_lifespan/_manager.py", line 92, in __aenter__
await self._exit_stack.aclose()
File "/usr/lib/python3.8/contextlib.py", line 621, in aclose
await self.__aexit__(None, None, None)
File "/usr/lib/python3.8/contextlib.py", line 679, in __aexit__
raise exc_details[1]
File "/usr/lib/python3.8/contextlib.py", line 662, in __aexit__
cb_suppress = await cb(*exc_details)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/asgi_lifespan/_concurrency/asyncio.py", line 80, in __aexit__
await self.task
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/asgi_lifespan/_manager.py", line 90, in __aenter__
await self.startup()
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/asgi_lifespan/_manager.py", line 36, in startup
raise self._app_exception
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/asgi_lifespan/_concurrency/asyncio.py", line 63, in run_and_silence_cancelled
await self.coroutine()
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/asgi_lifespan/_manager.py", line 64, in run_app
await self.app(scope, self.receive, self.send)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/fastapi/applications.py", line 269, in __call__
await super().__call__(scope, receive, send)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/starlette/applications.py", line 124, in __call__
await self.middleware_stack(scope, receive, send)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/starlette/middleware/errors.py", line 149, in __call__
await self.app(scope, receive, send)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/starlette/middleware/cors.py", line 76, in __call__
await self.app(scope, receive, send)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/starlette/exceptions.py", line 69, in __call__
await self.app(scope, receive, send)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/fastapi/middleware/asyncexitstack.py", line 21, in __call__
raise e
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/fastapi/middleware/asyncexitstack.py", line 18, in __call__
await self.app(scope, receive, send)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/starlette/routing.py", line 659, in __call__
await self.lifespan(scope, receive, send)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/starlette/routing.py", line 635, in lifespan
async with self.lifespan_context(app):
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/starlette/routing.py", line 530, in __aenter__
await self._router.startup()
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/starlette/routing.py", line 612, in startup
await handler()
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/prefect/orion/api/server.py", line 266, in run_migrations
await db.create_db()
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/prefect/orion/database/interface.py", line 53, in create_db
await self.run_migrations_upgrade()
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/prefect/orion/database/interface.py", line 61, in run_migrations_upgrade
await run_sync_in_worker_thread(alembic_upgrade)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/prefect/utilities/asyncutils.py", line 56, in run_sync_in_worker_thread
return await anyio.to_thread.run_sync(call, cancellable=True)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/anyio/to_thread.py", line 31, in run_sync
return await get_asynclib().run_sync_in_worker_thread(
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/anyio/_backends/_asyncio.py", line 937, in run_sync_in_worker_thread
return await future
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/anyio/_backends/_asyncio.py", line 867, in run
result = context.run(func, *args)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/prefect/orion/database/alembic_commands.py", line 29, in alembic_upgrade
alembic.command.upgrade(alembic_config(), revision, sql=dry_run)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/alembic/command.py", line 322, in upgrade
script.run_env()
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/alembic/script/base.py", line 569, in run_env
util.load_python_file(self.dir, "env.py")
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/alembic/util/pyfiles.py", line 94, in load_python_file
module = load_module_py(module_id, path)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/alembic/util/pyfiles.py", line 110, in load_module_py
spec.loader.exec_module(module) # type: ignore
File "<frozen importlib._bootstrap_external>", line 848, in exec_module
File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/prefect/orion/database/migrations/env.py", line 98, in <module>
apply_migrations()
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/prefect/utilities/asyncutils.py", line 189, in wrapper
return run_async_from_worker_thread(async_fn, *args, **kwargs)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/prefect/utilities/asyncutils.py", line 136, in run_async_from_worker_thread
return anyio.from_thread.run(call)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/anyio/from_thread.py", line 49, in run
return asynclib.run_async_from_thread(func, *args)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/anyio/_backends/_asyncio.py", line 970, in run_async_from_thread
return f.result()
File "/usr/lib/python3.8/concurrent/futures/_base.py", line 444, in result
return self.__get_result()
File "/usr/lib/python3.8/concurrent/futures/_base.py", line 389, in __get_result
raise self._exception
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/prefect/orion/database/migrations/env.py", line 92, in apply_migrations
await connection.run_sync(do_run_migrations)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/sqlalchemy/ext/asyncio/engine.py", line 546, in run_sync
return await greenlet_spawn(fn, conn, *arg, **kw)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 128, in greenlet_spawn
result = context.switch(value)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/prefect/orion/database/migrations/env.py", line 80, in do_run_migrations
context.run_migrations()
File "<string>", line 8, in run_migrations
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/alembic/runtime/environment.py", line 853, in run_migrations
self.get_context().run_migrations(**kw)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/alembic/runtime/migration.py", line 623, in run_migrations
step.migration_fn(**kw)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/prefect/orion/database/migrations/versions/sqlite/2022_05_30_100855_d76326ed0d06_rename_run_alerts_to_run_notifications.py", line 19, in upgrade
op.rename_table("flow_run_alert_policy", "flow_run_notification_policy")
File "<string>", line 8, in rename_table
File "<string>", line 3, in rename_table
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/alembic/operations/ops.py", line 1396, in rename_table
return operations.invoke(op)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/alembic/operations/base.py", line 394, in invoke
return fn(self, operation)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/alembic/operations/toimpl.py", line 122, in rename_table
operations.impl.rename_table(
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/alembic/ddl/impl.py", line 346, in rename_table
self._exec(
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/alembic/ddl/impl.py", line 195, in _exec
return conn.execute(construct, multiparams)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/sqlalchemy/future/engine.py", line 280, in execute
return self._execute_20(
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1631, in _execute_20
return meth(self, args_10style, kwargs_10style, execution_options)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/sqlalchemy/sql/ddl.py", line 80, in _execute_on_connection
return connection._execute_ddl(
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1398, in _execute_ddl
ret = self._execute_context(
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1862, in _execute_context
self._handle_dbapi_exception(
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 2043, in _handle_dbapi_exception
util.raise_(
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/sqlalchemy/util/compat.py", line 208, in raise_
raise exception
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1819, in _execute_context
self.dialect.do_execute(
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/sqlalchemy/engine/default.py", line 732, in do_execute
cursor.execute(statement, parameters)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/sqlalchemy/dialects/sqlite/aiosqlite.py", line 100, in execute
self._adapt_connection._handle_exception(error)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/sqlalchemy/dialects/sqlite/aiosqlite.py", line 228, in _handle_exception
raise error
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/sqlalchemy/dialects/sqlite/aiosqlite.py", line 82, in execute
self.await_(_cursor.execute(operation, parameters))
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 68, in await_only
return current.driver.switch(awaitable)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 121, in greenlet_spawn
value = await result
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/aiosqlite/cursor.py", line 37, in execute
await self._execute(self._cursor.execute, sql, parameters)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/aiosqlite/cursor.py", line 31, in _execute
return await self._conn._execute(fn, *args, **kwargs)
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/aiosqlite/core.py", line 129, in _execute
return await future
File "/home/rxrx/git/mousera2/build/venv/lib/python3.8/site-packages/aiosqlite/core.py", line 102, in run
result = function()
sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) no such table: flow_run_alert_policy
[SQL: ALTER TABLE flow_run_alert_policy RENAME TO flow_run_notification_policy]
(Background on this error at: https://sqlalche.me/e/14/e3q8)
About this issue
- Original URL
- State: closed
- Created 2 years ago
- Comments: 15 (1 by maintainers)
I worked around it by calling a subprocess of the prefect login command. Thanks for finding the error.
Also note this piece of code at the top of prefect_submit.py
Its required because prefect died on symlinked flows when bundling them as a Deployment. Unrelated, just a small issue.