如果你在实际安装或使用过程中,遇到一些其他问题,在文章中找不到答案,可留言一起探讨......
更多airflow资料,可查看:airflow从入门到精通学习笔记系列
问题1:airflow initdb报错
[root@VM_7_246_centos /data]# airflow initdb
Traceback (most recent call last):
File "/usr/local/bin/airflow", line 21, in <module>
from airflow import configuration
File "/usr/local/lib/python3.6/site-packages/airflow/__init__.py", line 40, in <module>
from flask_admin import BaseView
File "/usr/local/lib/python3.6/site-packages/flask_admin/__init__.py", line 6, in <module>
from .base import expose, expose_plugview, Admin, BaseView, AdminIndexView # noqa: F401
File "/usr/local/lib/python3.6/site-packages/flask_admin/base.py", line 6, in <module>
from flask import Blueprint, current_app, render_template, abort, g, url_for
File "/usr/local/lib64/python3.6/site-packages/flask/__init__.py", line 21, in <module>
from .app import Flask
File "/usr/local/lib64/python3.6/site-packages/flask/app.py", line 69, in <module>
from .wrappers import Request
File "/usr/local/lib64/python3.6/site-packages/flask/wrappers.py", line 14, in <module>
from werkzeug.wrappers.json import JSONMixin as _JSONMixin
ModuleNotFoundError: No module named 'werkzeug.wrappers.json'; 'werkzeug.wrappers' is not a package
解决办法:更改Flask版本
[root@VM_7_246_centos /data]# pip3 list | grep flask
flask-swagger (0.2.13)
[root@VM_7_246_centos /data]#
[root@VM_7_246_centos /data]# pip3 uninstall -y flask
Uninstalling Flask-1.1.1:
/usr/local/bin/flask
/usr/local/lib64/python3.6/site-packages/Flask-1.1.1.dist-info/INSTALLER
/usr/local/lib64/python3.6/site-packages/Flask-1.1.1.dist-info/LICENSE.rst
/usr/local/lib64/python3.6/site-packages/Flask-1.1.1.dist-info/METADATA
/usr/local/lib64/python3.6/site-packages/Flask-1.1.1.dist-info/RECORD
/usr/local/lib64/python3.6/site-packages/Flask-1.1.1.dist-info/WHEEL
/usr/local/lib64/python3.6/site-packages/Flask-1.1.1.dist-info/entry_points.txt
/usr/local/lib64/python3.6/site-packages/Flask-1.1.1.dist-info/top_level.txt
/usr/local/lib64/python3.6/site-packages/flask/__init__.py
/usr/local/lib64/python3.6/site-packages/flask/__main__.py
/usr/local/lib64/python3.6/site-packages/flask/__pycache__/__init__.cpython-36.pyc
/usr/local/lib64/python3.6/site-packages/flask/__pycache__/__main__.cpython-36.pyc
/usr/local/lib64/python3.6/site-packages/flask/__pycache__/_compat.cpython-36.pyc
/usr/local/lib64/python3.6/site-packages/flask/__pycache__/app.cpython-36.pyc
/usr/local/lib64/python3.6/site-packages/flask/__pycache__/blueprints.cpython-36.pyc
/usr/local/lib64/python3.6/site-packages/flask/__pycache__/cli.cpython-36.pyc
/usr/local/lib64/python3.6/site-packages/flask/__pycache__/config.cpython-36.pyc
/usr/local/lib64/python3.6/site-packages/flask/__pycache__/ctx.cpython-36.pyc
/usr/local/lib64/python3.6/site-packages/flask/__pycache__/debughelpers.cpython-36.pyc
/usr/local/lib64/python3.6/site-packages/flask/__pycache__/globals.cpython-36.pyc
/usr/local/lib64/python3.6/site-packages/flask/__pycache__/helpers.cpython-36.pyc
/usr/local/lib64/python3.6/site-packages/flask/__pycache__/logging.cpython-36.pyc
/usr/local/lib64/python3.6/site-packages/flask/__pycache__/sessions.cpython-36.pyc
/usr/local/lib64/python3.6/site-packages/flask/__pycache__/signals.cpython-36.pyc
/usr/local/lib64/python3.6/site-packages/flask/__pycache__/templating.cpython-36.pyc
/usr/local/lib64/python3.6/site-packages/flask/__pycache__/testing.cpython-36.pyc
/usr/local/lib64/python3.6/site-packages/flask/__pycache__/views.cpython-36.pyc
/usr/local/lib64/python3.6/site-packages/flask/__pycache__/wrappers.cpython-36.pyc
/usr/local/lib64/python3.6/site-packages/flask/_compat.py
/usr/local/lib64/python3.6/site-packages/flask/app.py
/usr/local/lib64/python3.6/site-packages/flask/blueprints.py
/usr/local/lib64/python3.6/site-packages/flask/cli.py
/usr/local/lib64/python3.6/site-packages/flask/config.py
/usr/local/lib64/python3.6/site-packages/flask/ctx.py
/usr/local/lib64/python3.6/site-packages/flask/debughelpers.py
/usr/local/lib64/python3.6/site-packages/flask/globals.py
/usr/local/lib64/python3.6/site-packages/flask/helpers.py
/usr/local/lib64/python3.6/site-packages/flask/json/__init__.py
/usr/local/lib64/python3.6/site-packages/flask/json/__pycache__/__init__.cpython-36.pyc
/usr/local/lib64/python3.6/site-packages/flask/json/__pycache__/tag.cpython-36.pyc
/usr/local/lib64/python3.6/site-packages/flask/json/tag.py
/usr/local/lib64/python3.6/site-packages/flask/logging.py
/usr/local/lib64/python3.6/site-packages/flask/sessions.py
/usr/local/lib64/python3.6/site-packages/flask/signals.py
/usr/local/lib64/python3.6/site-packages/flask/templating.py
/usr/local/lib64/python3.6/site-packages/flask/testing.py
/usr/local/lib64/python3.6/site-packages/flask/views.py
/usr/local/lib64/python3.6/site-packages/flask/wrappers.py
Successfully uninstalled Flask-1.1.1
[root@VM_7_246_centos /data]# pip3 install Flask==1.0.4
Collecting Flask==1.0.4
Downloading https://mirrors.cloud.tencent.com/pypi/packages/d8/94/7350820ae209ccdba073f83220cea1c376f2621254d1e0e82609c9a65e58/Flask-1.0.4-py2.py3-none-any.whl (92kB)
100% || 102kB 8.6MB/s
Requirement already satisfied (use --upgrade to upgrade): Werkzeug>=0.14 in /usr/local/lib64/python3.6/site-packages (from Flask==1.0.4)
Requirement already satisfied (use --upgrade to upgrade): itsdangerous>=0.24 in /usr/local/lib/python3.6/site-packages (from Flask==1.0.4)
Requirement already satisfied (use --upgrade to upgrade): click>=5.1 in /usr/local/lib64/python3.6/site-packages (from Flask==1.0.4)
Requirement already satisfied (use --upgrade to upgrade): Jinja2>=2.10 in /usr/local/lib64/python3.6/site-packages (from Flask==1.0.4)
Requirement already satisfied (use --upgrade to upgrade): MarkupSafe>=0.23 in /usr/local/lib64/python3.6/site-packages (from Jinja2>=2.10->Flask==1.0.4)
Installing collected packages: Flask
Successfully installed Flask-1.0.4
问题2:MySQL数据未配置explicit_defaults_for_timestamp参数
[root@VM_7_246_centos /data]# airflow initdb
[2019-08-05 19:45:28,916] {settings.py:182} INFO - settings.configure_orm(): Using pool settings. pool_size=5, pool_recycle=1800, pid=10006
[2019-08-05 19:45:29,025] {__init__.py:51} INFO - Using executor LocalExecutor
DB: mysql+pymysql://root:***@localhost:3306/airflow
[2019-08-05 19:45:29,400] {db.py:350} INFO - Creating tables
INFO [alembic.runtime.migration] Context impl MySQLImpl.
INFO [alembic.runtime.migration] Will assume non-transactional DDL.
INFO [alembic.runtime.migration] Running upgrade -> e3a246e0dc1, current schema
INFO [alembic.runtime.migration] Running upgrade e3a246e0dc1 -> 1507a7289a2f, create is_encrypted
INFO [alembic.runtime.migration] Running upgrade 1507a7289a2f -> 13eb55f81627, maintain history for compatibility with earlier migrations
INFO [alembic.runtime.migration] Running upgrade 13eb55f81627 -> 338e90f54d61, More logging into task_instance
INFO [alembic.runtime.migration] Running upgrade 338e90f54d61 -> 52d714495f0, job_id indices
INFO [alembic.runtime.migration] Running upgrade 52d714495f0 -> 502898887f84, Adding extra to Log
INFO [alembic.runtime.migration] Running upgrade 502898887f84 -> 1b38cef5b76e, add dagrun
INFO [alembic.runtime.migration] Running upgrade 1b38cef5b76e -> 2e541a1dcfed, task_duration
INFO [alembic.runtime.migration] Running upgrade 2e541a1dcfed -> 40e67319e3a9, dagrun_config
INFO [alembic.runtime.migration] Running upgrade 40e67319e3a9 -> 561833c1c74b, add password column to user
INFO [alembic.runtime.migration] Running upgrade 561833c1c74b -> 4446e08588, dagrun start end
INFO [alembic.runtime.migration] Running upgrade 4446e08588 -> bbc73705a13e, Add notification_sent column to sla_miss
INFO [alembic.runtime.migration] Running upgrade bbc73705a13e -> bba5a7cfc896, Add a column to track the encryption state of the 'Extra' field in connection
INFO [alembic.runtime.migration] Running upgrade bba5a7cfc896 -> 1968acfc09e3, add is_encrypted column to variable table
INFO [alembic.runtime.migration] Running upgrade 1968acfc09e3 -> 2e82aab8ef20, rename user table
INFO [alembic.runtime.migration] Running upgrade 2e82aab8ef20 -> 211e584da130, add TI state index
INFO [alembic.runtime.migration] Running upgrade 211e584da130 -> 64de9cddf6c9, add task fails journal table
INFO [alembic.runtime.migration] Running upgrade 64de9cddf6c9 -> f2ca10b85618, add dag_stats table
INFO [alembic.runtime.migration] Running upgrade f2ca10b85618 -> 4addfa1236f1, Add fractional seconds to mysql tables
INFO [alembic.runtime.migration] Running upgrade 4addfa1236f1 -> 8504051e801b, xcom dag task indices
INFO [alembic.runtime.migration] Running upgrade 8504051e801b -> 5e7d17757c7a, add pid field to TaskInstance
INFO [alembic.runtime.migration] Running upgrade 5e7d17757c7a -> 127d2bf2dfa7, Add dag_id/state index on dag_run table
INFO [alembic.runtime.migration] Running upgrade 127d2bf2dfa7 -> cc1e65623dc7, add max tries column to task instance
INFO [alembic.runtime.migration] Running upgrade cc1e65623dc7 -> bdaa763e6c56, Make xcom value column a large binary
INFO [alembic.runtime.migration] Running upgrade bdaa763e6c56 -> 947454bf1dff, add ti job_id index
INFO [alembic.runtime.migration] Running upgrade 947454bf1dff -> d2ae31099d61, Increase text size for MySQL (not relevant for other DBs' text types)
INFO [alembic.runtime.migration] Running upgrade d2ae31099d61 -> 0e2a74e0fc9f, Add time zone awareness
Traceback (most recent call last):
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1236, in _execute_context
cursor, statement, parameters, context
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/default.py", line 536, in do_execute
cursor.execute(statement, parameters)
File "/usr/local/lib64/python3.6/site-packages/pymysql/cursors.py", line 170, in execute
result = self._query(query)
File "/usr/local/lib64/python3.6/site-packages/pymysql/cursors.py", line 328, in _query
conn.query(q)
File "/usr/local/lib64/python3.6/site-packages/pymysql/connections.py", line 517, in query
self._affected_rows = self._read_query_result(unbuffered=unbuffered)
File "/usr/local/lib64/python3.6/site-packages/pymysql/connections.py", line 732, in _read_query_result
result.read()
File "/usr/local/lib64/python3.6/site-packages/pymysql/connections.py", line 1075, in read
first_packet = self.connection._read_packet()
File "/usr/local/lib64/python3.6/site-packages/pymysql/connections.py", line 684, in _read_packet
packet.check_error()
File "/usr/local/lib64/python3.6/site-packages/pymysql/protocol.py", line 220, in check_error
err.raise_mysql_exception(self._data)
File "/usr/local/lib64/python3.6/site-packages/pymysql/err.py", line 109, in raise_mysql_exception
raise errorclass(errno, errval)
pymysql.err.InternalError: (1193, "Unknown system variable 'explicit_defaults_for_timestamp'")
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/bin/airflow", line 32, in <module>
args.func(args)
File "/usr/local/lib/python3.6/site-packages/airflow/bin/cli.py", line 1096, in initdb
db.initdb(settings.RBAC)
File "/usr/local/lib/python3.6/site-packages/airflow/utils/db.py", line 91, in initdb
upgradedb()
File "/usr/local/lib/python3.6/site-packages/airflow/utils/db.py", line 358, in upgradedb
command.upgrade(config, 'heads')
File "/usr/local/lib/python3.6/site-packages/alembic/command.py", line 254, in upgrade
script.run_env()
File "/usr/local/lib/python3.6/site-packages/alembic/script/base.py", line 427, in run_env
util.load_python_file(self.dir, 'env.py')
File "/usr/local/lib/python3.6/site-packages/alembic/util/pyfiles.py", line 81, in load_python_file
module = load_module_py(module_id, path)
File "/usr/local/lib/python3.6/site-packages/alembic/util/compat.py", line 83, in load_module_py
spec.loader.exec_module(module)
File "<frozen importlib._bootstrap_external>", line 678, in exec_module
File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
File "/usr/local/lib/python3.6/site-packages/airflow/migrations/env.py", line 92, in <module>
run_migrations_online()
File "/usr/local/lib/python3.6/site-packages/airflow/migrations/env.py", line 86, in run_migrations_online
context.run_migrations()
File "<string>", line 8, in run_migrations
File "/usr/local/lib/python3.6/site-packages/alembic/runtime/environment.py", line 836, in run_migrations
self.get_context().run_migrations(**kw)
File "/usr/local/lib/python3.6/site-packages/alembic/runtime/migration.py", line 330, in run_migrations
step.migration_fn(**kw)
File "/usr/local/lib/python3.6/site-packages/airflow/migrations/versions/0e2a74e0fc9f_add_time_zone_awareness.py", line 43, in upgrade
cur = conn.execute("SELECT @@explicit_defaults_for_timestamp")
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 974, in execute
return self._execute_text(object_, multiparams, params)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1147, in _execute_text
parameters,
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1240, in _execute_context
e, statement, parameters, cursor, context
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1458, in _handle_dbapi_exception
util.raise_from_cause(sqlalchemy_exception, exc_info)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/compat.py", line 296, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/compat.py", line 276, in reraise
raise value.with_traceback(tb)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1236, in _execute_context
cursor, statement, parameters, context
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/default.py", line 536, in do_execute
cursor.execute(statement, parameters)
File "/usr/local/lib64/python3.6/site-packages/pymysql/cursors.py", line 170, in execute
result = self._query(query)
File "/usr/local/lib64/python3.6/site-packages/pymysql/cursors.py", line 328, in _query
conn.query(q)
File "/usr/local/lib64/python3.6/site-packages/pymysql/connections.py", line 517, in query
self._affected_rows = self._read_query_result(unbuffered=unbuffered)
File "/usr/local/lib64/python3.6/site-packages/pymysql/connections.py", line 732, in _read_query_result
result.read()
File "/usr/local/lib64/python3.6/site-packages/pymysql/connections.py", line 1075, in read
first_packet = self.connection._read_packet()
File "/usr/local/lib64/python3.6/site-packages/pymysql/connections.py", line 684, in _read_packet
packet.check_error()
File "/usr/local/lib64/python3.6/site-packages/pymysql/protocol.py", line 220, in check_error
err.raise_mysql_exception(self._data)
File "/usr/local/lib64/python3.6/site-packages/pymysql/err.py", line 109, in raise_mysql_exception
raise errorclass(errno, errval)
sqlalchemy.exc.InternalError: (pymysql.err.InternalError) (1193, "Unknown system variable 'explicit_defaults_for_timestamp'") [SQL: 'SELECT @@explicit_defaults_for_timestamp'] (Background on this error at: http://sqlalche.me/e/2j85)
解决办法:更改MySQL配置
vim /etc/my.cnf
[mysqld]
explicit_defaults_for_timestamp=1
问题3:'Pendulum' object has no attribute 'translate'
[root@VM_7_246_centos /data/airflow]# airflow test tutorial print_date 2015-06-01
[2019-08-05 21:42:21,343] {settings.py:182} INFO - settings.configure_orm(): Using pool settings. pool_size=5, pool_recycle=1800, pid=7027
[2019-08-05 21:42:21,451] {__init__.py:51} INFO - Using executor LocalExecutor
[2019-08-05 21:42:21,798] {cli_action_loggers.py:70} ERROR - Failed on pre-execution callback using <function default_action_log at 0x7f23ba2678c8>
Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/airflow/utils/cli_action_loggers.py", line 68, in on_pre_execution
cb(**kwargs)
File "/usr/local/lib/python3.6/site-packages/airflow/utils/cli_action_loggers.py", line 99, in default_action_log
session.add(log)
File "/usr/lib64/python3.6/contextlib.py", line 88, in __exit__
next(self.gen)
File "/usr/local/lib/python3.6/site-packages/airflow/utils/db.py", line 44, in create_session
session.commit()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/orm/session.py", line 1023, in commit
self.transaction.commit()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/orm/session.py", line 487, in commit
self._prepare_impl()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/orm/session.py", line 466, in _prepare_impl
self.session.flush()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/orm/session.py", line 2446, in flush
self._flush(objects)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/orm/session.py", line 2584, in _flush
transaction.rollback(_capture_exception=True)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/langhelpers.py", line 67, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/compat.py", line 277, in reraise
raise value
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/orm/session.py", line 2544, in _flush
flush_context.execute()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/orm/unitofwork.py", line 416, in execute
rec.execute(self)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/orm/unitofwork.py", line 583, in execute
uow,
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/orm/persistence.py", line 245, in save_obj
insert,
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/orm/persistence.py", line 1116, in _emit_insert_statements
statement, params
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 980, in execute
return meth(self, multiparams, params)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/sql/elements.py", line 273, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1099, in _execute_clauseelement
distilled_params,
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1240, in _execute_context
e, statement, parameters, cursor, context
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1460, in _handle_dbapi_exception
util.reraise(*exc_info)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/compat.py", line 277, in reraise
raise value
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1236, in _execute_context
cursor, statement, parameters, context
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/default.py", line 536, in do_execute
cursor.execute(statement, parameters)
File "/usr/local/lib64/python3.6/site-packages/pymysql/cursors.py", line 168, in execute
query = self.mogrify(query, args)
File "/usr/local/lib64/python3.6/site-packages/pymysql/cursors.py", line 147, in mogrify
query = query % self._escape_args(args, conn)
File "/usr/local/lib64/python3.6/site-packages/pymysql/cursors.py", line 127, in _escape_args
return {key: conn.literal(val) for (key, val) in args.items()}
File "/usr/local/lib64/python3.6/site-packages/pymysql/cursors.py", line 127, in <dictcomp>
return {key: conn.literal(val) for (key, val) in args.items()}
File "/usr/local/lib64/python3.6/site-packages/pymysql/connections.py", line 467, in literal
return self.escape(obj, self.encoders)
File "/usr/local/lib64/python3.6/site-packages/pymysql/connections.py", line 460, in escape
return converters.escape_item(obj, self.charset, mapping=mapping)
File "/usr/local/lib64/python3.6/site-packages/pymysql/converters.py", line 27, in escape_item
val = encoder(val, mapping)
File "/usr/local/lib64/python3.6/site-packages/pymysql/converters.py", line 118, in escape_unicode
return u"'%s'" % _escape_unicode(value)
File "/usr/local/lib64/python3.6/site-packages/pymysql/converters.py", line 73, in _escape_unicode
return value.translate(_escape_table)
AttributeError: 'Pendulum' object has no attribute 'translate'
解决办法:
vim /usr/local/lib/python3.6/site-packages/airflow/settings.py +226
def configure_adapters():
try:
#import MySQLdb.converters
#MySQLdb.converters.conversions[Pendulum] = MySQLdb.converters.DateTime2literal
import pymysql.converters
pymysql.converters.conversions[Pendulum] = pymysql.converters.escape_datetime
except ImportError:
pass
更多airflow资料,可查看:airflow从入门到精通学习笔记系列
如发现文中有错误,望留言指明,万分感激;
如对此文章内容感兴趣,想进一步探讨,可以留言交流;
如想转发此文章,请留言协商一下,切勿不指明出处的转发,尊重原创;
如阅读过程中有收获,并想感谢一下,欢迎打赏;
----小林帮