Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ jobs:
integration-tests:
strategy:
matrix:
dialect: [mysql, postgresql, sqlite, mssql]
dialect: [mysql, postgresql, sqlite, mssql, clickhouse]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
Expand Down
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,7 @@ The provider supports the following databases:
* PostgreSQL
* SQLite
* Microsoft SQL Server
* ClickHouse

### FAQ

Expand Down
1 change: 1 addition & 0 deletions atlas_provider_sqlalchemy/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ class Dialect(str, Enum):
postgresql = "postgresql"
sqlite = "sqlite"
mssql = "mssql"
clickhouse = "clickhouse"


def run(
Expand Down
645 changes: 643 additions & 2 deletions poetry.lock

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ packages = [{include = "atlas_provider_sqlalchemy"}]
python = "^3.9"
typer = ">0.9.0,<=1.0.0"
sqlalchemy = "^2.0.41"
clickhouse-sqlalchemy = "^0.3.2"

[tool.pyrefly]
project-includes = [
Expand Down
3 changes: 2 additions & 1 deletion tests/atlas-script.hcl
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ locals {
postgresql = "docker://postgres/15"
sqlite = "sqlite://?mode=memory&_fk=1"
mssql = "docker://sqlserver/2022-latest"
clickhouse = "docker://clickhouse/25.9/dev"
}[var.dialect]
}

Expand All @@ -17,7 +18,7 @@ data "external_schema" "sqlalchemy" {
"run",
"python3",
"load_models.py",
var.dialect, // mysql | postgresql | sqlite | mssql
var.dialect, // mysql | postgresql | sqlite | mssql | clickhouse
]
}

Expand Down
3 changes: 2 additions & 1 deletion tests/atlas-standalone.hcl
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ locals {
postgresql = "docker://postgres/15"
sqlite = "sqlite://?mode=memory&_fk=1"
mssql = "docker://sqlserver/2022-latest"
clickhouse = "docker://clickhouse/23.11/dev"
}[var.dialect]
}

Expand All @@ -23,7 +24,7 @@ data "external_schema" "sqlalchemy" {
"python3",
"../atlas_provider_sqlalchemy/main.py",
"--path", var.path,
"--dialect", var.dialect, // mysql | postgresql | sqlite | mssql
"--dialect", var.dialect, // mysql | postgresql | sqlite | mssql | clickhouse
]
}

Expand Down
14 changes: 14 additions & 0 deletions tests/migrations/clickhouse/20251007100727.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
-- Create "address" table
CREATE TABLE `address` (
`id` Int32,
`email_address` FixedString(30),
`user_id` Int32
) ENGINE = MergeTree
PRIMARY KEY (`id`) ORDER BY (`id`) SETTINGS index_granularity = 8192;
-- Create "user_account" table
CREATE TABLE `user_account` (
`id` Int32,
`name` FixedString(30),
`fullname` FixedString(30)
) ENGINE = MergeTree
PRIMARY KEY (`id`) ORDER BY (`id`) SETTINGS index_granularity = 8192;
2 changes: 2 additions & 0 deletions tests/migrations/clickhouse/atlas.sum
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
h1:erjqmA/a6B0TjDxdO09+CH0ToQUBXoN9Nhos0cswMpc=
20251007100727.sql h1:sOg0sT6jinsVMTDsIeArCpUIcljWYu4lmBqLPuBKGQo=
6 changes: 5 additions & 1 deletion tests/test_main.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
[
(Dialect.postgresql, "tests/testdata/models/ddl_postgres.sql"),
(Dialect.mysql, "tests/testdata/models/ddl_mysql.sql"),
(Dialect.clickhouse, "tests/testdata/models/ddl_clickhouse.sql"),
],
)
def test_run_models(
Expand All @@ -44,6 +45,7 @@ def test_run_models(
[
(Dialect.postgresql, "tests/testdata/old_models/ddl_postgres.sql"),
(Dialect.mysql, "tests/testdata/old_models/ddl_mysql.sql"),
(Dialect.clickhouse, "tests/testdata/old_models/ddl_clickhouse.sql"),
],
)
def test_run_old_models(
Expand All @@ -65,6 +67,7 @@ def test_run_old_models(
[
(Dialect.postgresql, "tests/testdata/structured_models/ddl_postgres.sql"),
(Dialect.mysql, "tests/testdata/structured_models/ddl_mysql.sql"),
(Dialect.clickhouse, "tests/testdata/structured_models/ddl_clickhouse.sql"),
],
)
def test_run_structured_models(
Expand All @@ -83,7 +86,7 @@ def test_run_structured_models(

@pytest.mark.parametrize(
"dialect",
[Dialect.postgresql, Dialect.mysql],
[Dialect.postgresql, Dialect.mysql, Dialect.clickhouse],
)
def test_run_multiple_paths(
dialect: Dialect,
Expand All @@ -109,6 +112,7 @@ def test_run_multiple_paths(
[
(Dialect.postgresql, "tests/testdata/tables/ddl_postgres.sql"),
(Dialect.mysql, "tests/testdata/tables/ddl_mysql.sql"),
(Dialect.clickhouse, "tests/testdata/tables/ddl_clickhouse.sql"),
],
)
def test_run_models_2(
Expand Down
7 changes: 7 additions & 0 deletions tests/testdata/models/ddl_clickhouse.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
-- atlas:pos user_account[type=table] [ABS_PATH]/tests/testdata/models/models.py:11
-- atlas:pos address[type=table] [ABS_PATH]/tests/testdata/models/models.py:26

CREATE TABLE user_account (id INTEGER, name FixedString(30), fullname FixedString(30)) ENGINE = MergeTree() ORDER BY id;

CREATE TABLE address (id INTEGER, email_address FixedString(30), user_id INTEGER) ENGINE = MergeTree() ORDER BY id;

4 changes: 2 additions & 2 deletions tests/testdata/models/ddl_mysql.sql
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
-- atlas:pos user_account[type=table] [ABS_PATH]/tests/testdata/models/models.py:10
-- atlas:pos address[type=table] [ABS_PATH]/tests/testdata/models/models.py:23
-- atlas:pos user_account[type=table] [ABS_PATH]/tests/testdata/models/models.py:11
-- atlas:pos address[type=table] [ABS_PATH]/tests/testdata/models/models.py:26

CREATE TABLE user_account (id INTEGER NOT NULL AUTO_INCREMENT, name VARCHAR(30) NOT NULL, fullname VARCHAR(30), PRIMARY KEY (id));

Expand Down
4 changes: 2 additions & 2 deletions tests/testdata/models/ddl_postgres.sql
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
-- atlas:pos user_account[type=table] [ABS_PATH]/tests/testdata/models/models.py:10
-- atlas:pos address[type=table] [ABS_PATH]/tests/testdata/models/models.py:23
-- atlas:pos user_account[type=table] [ABS_PATH]/tests/testdata/models/models.py:11
-- atlas:pos address[type=table] [ABS_PATH]/tests/testdata/models/models.py:26

CREATE TABLE user_account (id SERIAL NOT NULL, name VARCHAR(30) NOT NULL, fullname VARCHAR(30), PRIMARY KEY (id));

Expand Down
5 changes: 5 additions & 0 deletions tests/testdata/models/models.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from typing import List, Optional
from sqlalchemy import ForeignKey, String
from sqlalchemy.orm import Mapped, mapped_column, relationship, DeclarativeBase
from clickhouse_sqlalchemy import engines


class Base(DeclarativeBase):
Expand All @@ -9,6 +10,8 @@ class Base(DeclarativeBase):

class User(Base):
__tablename__ = "user_account"
__table_args__ = (engines.MergeTree(order_by="id"),)

id: Mapped[int] = mapped_column(primary_key=True)
name: Mapped[str] = mapped_column(String(30))
fullname: Mapped[Optional[str]] = mapped_column(String(30))
Expand All @@ -22,6 +25,8 @@ def __repr__(self) -> str:

class Address(Base):
__tablename__ = "address"
__table_args__ = (engines.MergeTree(order_by="id"),)

id: Mapped[int] = mapped_column(primary_key=True)
email_address: Mapped[str] = mapped_column(String(30))
user_id: Mapped[int] = mapped_column(ForeignKey("user_account.id"))
Expand Down
13 changes: 13 additions & 0 deletions tests/testdata/multi_path/ddl_clickhouse.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
-- atlas:pos user_account[type=table] [ABS_PATH]/tests/testdata/multi_path/models1/models.py:10
-- atlas:pos address[type=table] [ABS_PATH]/tests/testdata/multi_path/models1/models.py:21
-- atlas:pos user_account2[type=table] [ABS_PATH]/tests/testdata/multi_path/models2/models.py:10
-- atlas:pos address2[type=table] [ABS_PATH]/tests/testdata/multi_path/models2/models.py:22

CREATE TABLE user_account (id INTEGER, name FixedString(30), fullname FixedString(30)) ENGINE = MergeTree() ORDER BY id;

CREATE TABLE address (id INTEGER, email_address FixedString(30), user_id INTEGER) ENGINE = MergeTree() ORDER BY id;

CREATE TABLE user_account2 (id INTEGER, name FixedString(30), fullname FixedString(30)) ENGINE = MergeTree() ORDER BY id;

CREATE TABLE address2 (id INTEGER, email_address FixedString(30), user_id INTEGER) ENGINE = MergeTree() ORDER BY id;

8 changes: 4 additions & 4 deletions tests/testdata/multi_path/ddl_mysql.sql
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
-- atlas:pos user_account[type=table] [ABS_PATH]/tests/testdata/multi_path/models1/models.py:9
-- atlas:pos address[type=table] [ABS_PATH]/tests/testdata/multi_path/models1/models.py:18
-- atlas:pos user_account2[type=table] [ABS_PATH]/tests/testdata/multi_path/models2/models.py:9
-- atlas:pos address2[type=table] [ABS_PATH]/tests/testdata/multi_path/models2/models.py:19
-- atlas:pos user_account[type=table] [ABS_PATH]/tests/testdata/multi_path/models1/models.py:10
-- atlas:pos address[type=table] [ABS_PATH]/tests/testdata/multi_path/models1/models.py:21
-- atlas:pos user_account2[type=table] [ABS_PATH]/tests/testdata/multi_path/models2/models.py:10
-- atlas:pos address2[type=table] [ABS_PATH]/tests/testdata/multi_path/models2/models.py:22

CREATE TABLE user_account (id INTEGER NOT NULL AUTO_INCREMENT, name VARCHAR(30) NOT NULL, fullname VARCHAR(30), PRIMARY KEY (id));

Expand Down
8 changes: 4 additions & 4 deletions tests/testdata/multi_path/ddl_postgresql.sql
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
-- atlas:pos user_account[type=table] [ABS_PATH]/tests/testdata/multi_path/models1/models.py:9
-- atlas:pos address[type=table] [ABS_PATH]/tests/testdata/multi_path/models1/models.py:18
-- atlas:pos user_account2[type=table] [ABS_PATH]/tests/testdata/multi_path/models2/models.py:9
-- atlas:pos address2[type=table] [ABS_PATH]/tests/testdata/multi_path/models2/models.py:19
-- atlas:pos user_account[type=table] [ABS_PATH]/tests/testdata/multi_path/models1/models.py:10
-- atlas:pos address[type=table] [ABS_PATH]/tests/testdata/multi_path/models1/models.py:21
-- atlas:pos user_account2[type=table] [ABS_PATH]/tests/testdata/multi_path/models2/models.py:10
-- atlas:pos address2[type=table] [ABS_PATH]/tests/testdata/multi_path/models2/models.py:22

CREATE TABLE user_account (id SERIAL NOT NULL, name VARCHAR(30) NOT NULL, fullname VARCHAR(30), PRIMARY KEY (id));

Expand Down
5 changes: 5 additions & 0 deletions tests/testdata/multi_path/models1/models.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from sqlalchemy import ForeignKey, String
from sqlalchemy.orm import Mapped, mapped_column, relationship, DeclarativeBase
from clickhouse_sqlalchemy import engines


class Base(DeclarativeBase):
Expand All @@ -8,6 +9,8 @@ class Base(DeclarativeBase):

class User(Base):
__tablename__ = "user_account"
__table_args__ = (engines.MergeTree(order_by="id"),)

id: Mapped[int] = mapped_column(primary_key=True)
name: Mapped[str] = mapped_column(String(30))
fullname: Mapped[str | None] = mapped_column(String(30))
Expand All @@ -17,6 +20,8 @@ class User(Base):

class Address(Base):
__tablename__ = "address"
__table_args__ = (engines.MergeTree(order_by="id"),)

id: Mapped[int] = mapped_column(primary_key=True)
email_address: Mapped[str] = mapped_column(String(30))
user_id: Mapped[int] = mapped_column(ForeignKey("user_account.id"))
Expand Down
5 changes: 5 additions & 0 deletions tests/testdata/multi_path/models2/models.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from sqlalchemy import ForeignKey, String
from sqlalchemy.orm import Mapped, mapped_column, relationship, DeclarativeBase
from clickhouse_sqlalchemy import engines


class Base(DeclarativeBase):
Expand All @@ -8,6 +9,8 @@ class Base(DeclarativeBase):

class User2(Base):
__tablename__ = "user_account2"
__table_args__ = (engines.MergeTree(order_by="id"),)

id: Mapped[int] = mapped_column(primary_key=True)
name: Mapped[str] = mapped_column(String(30))
fullname: Mapped[str | None] = mapped_column(String(30))
Expand All @@ -18,6 +21,8 @@ class User2(Base):

class Address2(Base):
__tablename__ = "address2"
__table_args__ = (engines.MergeTree(order_by="id"),)

id: Mapped[int] = mapped_column(primary_key=True)
email_address: Mapped[str] = mapped_column(String(30))
user_id: Mapped[int] = mapped_column(ForeignKey("user_account2.id"))
Expand Down
7 changes: 7 additions & 0 deletions tests/testdata/old_models/ddl_clickhouse.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
-- atlas:pos user_account[type=table] [ABS_PATH]/tests/testdata/old_models/models.py:11
-- atlas:pos address[type=table] [ABS_PATH]/tests/testdata/old_models/models.py:23

CREATE TABLE user_account (id INTEGER, name FixedString(30), fullname FixedString(30)) ENGINE = MergeTree() ORDER BY id;

CREATE TABLE address (id INTEGER, email_address FixedString(30), user_id INTEGER) ENGINE = MergeTree() ORDER BY id;

4 changes: 2 additions & 2 deletions tests/testdata/old_models/ddl_mysql.sql
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
-- atlas:pos user_account[type=table] [ABS_PATH]/tests/testdata/old_models/models.py:10
-- atlas:pos address[type=table] [ABS_PATH]/tests/testdata/old_models/models.py:20
-- atlas:pos user_account[type=table] [ABS_PATH]/tests/testdata/old_models/models.py:11
-- atlas:pos address[type=table] [ABS_PATH]/tests/testdata/old_models/models.py:23

CREATE TABLE user_account (id INTEGER NOT NULL AUTO_INCREMENT, name VARCHAR(30) NOT NULL, fullname VARCHAR(30), PRIMARY KEY (id));

Expand Down
4 changes: 2 additions & 2 deletions tests/testdata/old_models/ddl_postgres.sql
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
-- atlas:pos user_account[type=table] [ABS_PATH]/tests/testdata/old_models/models.py:10
-- atlas:pos address[type=table] [ABS_PATH]/tests/testdata/old_models/models.py:20
-- atlas:pos user_account[type=table] [ABS_PATH]/tests/testdata/old_models/models.py:11
-- atlas:pos address[type=table] [ABS_PATH]/tests/testdata/old_models/models.py:23

CREATE TABLE user_account (id SERIAL NOT NULL, name VARCHAR(30) NOT NULL, fullname VARCHAR(30), PRIMARY KEY (id));

Expand Down
5 changes: 5 additions & 0 deletions tests/testdata/old_models/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,16 @@

from sqlalchemy.orm import relationship
from sqlalchemy.ext.declarative import declarative_base
from clickhouse_sqlalchemy import engines

# Using the old way of declaring a declarative base
Base = declarative_base()


class User(Base):
__tablename__ = "user_account"
__table_args__ = (engines.MergeTree(order_by="id"),)

id = Column(Integer, primary_key=True)
name = Column(String(30), nullable=False)
fullname = Column(String(30))
Expand All @@ -19,6 +22,8 @@ class User(Base):

class Address(Base):
__tablename__ = "address"
__table_args__ = (engines.MergeTree(order_by="id"),)

id = Column(Integer, primary_key=True)
email_address = Column(String(30), nullable=False)
user_id = Column(ForeignKey("user_account.id"), nullable=False)
Expand Down
7 changes: 7 additions & 0 deletions tests/testdata/structured_models/ddl_clickhouse.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
-- atlas:pos user_account[type=table] [ABS_PATH]/tests/testdata/structured_models/models/user.py:8
-- atlas:pos address[type=table] [ABS_PATH]/tests/testdata/structured_models/models/user.py:23

CREATE TABLE user_account (id INTEGER, name FixedString(30), fullname FixedString(30)) ENGINE = MergeTree() ORDER BY id;

CREATE TABLE address (id INTEGER, email_address FixedString(30), user_id INTEGER) ENGINE = MergeTree() ORDER BY id;

4 changes: 2 additions & 2 deletions tests/testdata/structured_models/ddl_mysql.sql
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
-- atlas:pos user_account[type=table] [ABS_PATH]/tests/testdata/structured_models/models/user.py:7
-- atlas:pos address[type=table] [ABS_PATH]/tests/testdata/structured_models/models/user.py:20
-- atlas:pos user_account[type=table] [ABS_PATH]/tests/testdata/structured_models/models/user.py:8
-- atlas:pos address[type=table] [ABS_PATH]/tests/testdata/structured_models/models/user.py:23

CREATE TABLE user_account (id INTEGER NOT NULL AUTO_INCREMENT, name VARCHAR(30) NOT NULL, fullname VARCHAR(30), PRIMARY KEY (id));

Expand Down
4 changes: 2 additions & 2 deletions tests/testdata/structured_models/ddl_postgres.sql
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
-- atlas:pos user_account[type=table] [ABS_PATH]/tests/testdata/structured_models/models/user.py:7
-- atlas:pos address[type=table] [ABS_PATH]/tests/testdata/structured_models/models/user.py:20
-- atlas:pos user_account[type=table] [ABS_PATH]/tests/testdata/structured_models/models/user.py:8
-- atlas:pos address[type=table] [ABS_PATH]/tests/testdata/structured_models/models/user.py:23

CREATE TABLE user_account (id SERIAL NOT NULL, name VARCHAR(30) NOT NULL, fullname VARCHAR(30), PRIMARY KEY (id));

Expand Down
5 changes: 5 additions & 0 deletions tests/testdata/structured_models/models/user.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
from typing import List, Optional
from sqlalchemy import String, ForeignKey
from sqlalchemy.orm import Mapped, mapped_column, relationship
from clickhouse_sqlalchemy import engines

from tests.testdata.structured_models.base import Base

class User(Base):
__tablename__ = "user_account"
__table_args__ = (engines.MergeTree(order_by="id"),)

id: Mapped[int] = mapped_column(primary_key=True)
name: Mapped[str] = mapped_column(String(30))
fullname: Mapped[Optional[str]] = mapped_column(String(30))
Expand All @@ -19,6 +22,8 @@ def __repr__(self) -> str:

class Address(Base):
__tablename__ = "address"
__table_args__ = (engines.MergeTree(order_by="id"),)

id: Mapped[int] = mapped_column(primary_key=True)
email_address: Mapped[str] = mapped_column(String(30))
user_id: Mapped[int] = mapped_column(ForeignKey("user_account.id"))
Expand Down
7 changes: 7 additions & 0 deletions tests/testdata/tables/ddl_clickhouse.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
-- atlas:pos user_account[type=table] [ABS_PATH]/tests/testdata/tables/tables.py:6
-- atlas:pos address[type=table] [ABS_PATH]/tests/testdata/tables/tables.py:15

CREATE TABLE user_account (id INTEGER, name FixedString(30), fullname FixedString(30)) ENGINE = MergeTree() ORDER BY id;

CREATE TABLE address (id INTEGER, email_address FixedString(30), user_id INTEGER) ENGINE = MergeTree() ORDER BY id;

4 changes: 2 additions & 2 deletions tests/testdata/tables/ddl_mysql.sql
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
-- atlas:pos user_account[type=table] [ABS_PATH]/tests/testdata/tables/tables.py:5
-- atlas:pos address[type=table] [ABS_PATH]/tests/testdata/tables/tables.py:13
-- atlas:pos user_account[type=table] [ABS_PATH]/tests/testdata/tables/tables.py:6
-- atlas:pos address[type=table] [ABS_PATH]/tests/testdata/tables/tables.py:15

CREATE TABLE user_account (id INTEGER NOT NULL AUTO_INCREMENT, name VARCHAR(30) NOT NULL, fullname VARCHAR(30), PRIMARY KEY (id));

Expand Down
4 changes: 2 additions & 2 deletions tests/testdata/tables/ddl_postgres.sql
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
-- atlas:pos user_account[type=table] [ABS_PATH]/tests/testdata/tables/tables.py:5
-- atlas:pos address[type=table] [ABS_PATH]/tests/testdata/tables/tables.py:13
-- atlas:pos user_account[type=table] [ABS_PATH]/tests/testdata/tables/tables.py:6
-- atlas:pos address[type=table] [ABS_PATH]/tests/testdata/tables/tables.py:15

CREATE TABLE user_account (id SERIAL NOT NULL, name VARCHAR(30) NOT NULL, fullname VARCHAR(30), PRIMARY KEY (id));

Expand Down
Loading