Added chunk size parameter to MakeDb for backed mode.

Author: kennypavan

README.md

@@ -221,6 +221,7 @@ adata_sql.query("SELECT corr(ITGB2,SSU72) as correlation FROM adata WHERE bulk_l
           <li><code>adata</code>: AnnData object (required)</li>
           <li><code>db_name</code>: Name for the database (required)</li>
           <li><code>db_path</code>: Path to store the database (default: 'db/')</li>
+		  <li><code>chunk_size</code>: When opening AnnData in backed mode, the amount of chunks to insert at one time. Lower the value for low-memory systems (default: 5000)</li>
 		  <li><code>layers</code>: List (optional. default: ["X", "obs", "var", "var_names", "obsm", "varm", "obsp", "uns"]).<i>The layers of the Anndata object to build into the database. For larger datasets, it may be beneficial to only include the layers you're interested in querying.</i></li>
           <li><code>create_basic_indexes</code>: Build indexed on cell_id (optional. default: False)</li>
 		  <li><code>create_all_indexes</code>: Boolean (optional. default: False). <i>Warning: Runtime can be significant when building.</i></li>

setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name='AnnSQL',
-    version='v0.9.1',
+    version='v0.9.2',
     author="Kenny Pavan",
     author_email="pavan@ohsu.edu",
     description="A Python SQL tool for converting Anndata objects to a relational DuckDb database. Methods are included for querying and basic single-cell preprocessing (experimental). ",

src/AnnSQL.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: AnnSQL
-Version: 0.9.1
+Version: 0.9.2
 Summary: A Python SQL tool for converting Anndata objects to a relational DuckDb database. Methods are included for querying and basic single-cell preprocessing (experimental). 
 Home-page: https://github.com/kennypavan/AnnSQL
 Author: Kenny Pavan
@@ -237,6 +237,7 @@ adata_sql.query("SELECT corr(ITGB2,SSU72) as correlation FROM adata WHERE bulk_l
           <li><code>adata</code>: AnnData object (required)</li>
           <li><code>db_name</code>: Name for the database (required)</li>
           <li><code>db_path</code>: Path to store the database (default: 'db/')</li>
+		  <li><code>chunk_size</code>: When opening AnnData in backed mode, the amount of chunks to insert at one time. Lower the value for low-memory systems (default: 5000)</li>
 		  <li><code>layers</code>: List (optional. default: ["X", "obs", "var", "var_names", "obsm", "varm", "obsp", "uns"]).<i>The layers of the Anndata object to build into the database. For larger datasets, it may be beneficial to only include the layers you're interested in querying.</i></li>
           <li><code>create_basic_indexes</code>: Build indexed on cell_id (optional. default: False)</li>
 		  <li><code>create_all_indexes</code>: Boolean (optional. default: False). <i>Warning: Runtime can be significant when building.</i></li>

src/AnnSQL/BuildDb.py

@@ -25,13 +25,15 @@ def __init__(self,
 				create_all_indexes=False, 
 				create_basic_indexes=False,
 				convenience_view=True,
+				chunk_size=5000,
 				layers=["X", "obs", "var", "var_names", "obsm", "varm", "obsp", "uns"]):
 		self.adata = adata
 		self.conn = conn
 		self.create_all_indexes = create_all_indexes
 		self.create_basic_indexes = create_basic_indexes
 		self.convenience_view = convenience_view
 		self.layers = layers
+		self.chunk_size = chunk_size
 		self.build()
 		if "uns" in self.layers: #not recommended for large datasets
 			self.build_uns_layer()
@@ -76,7 +78,7 @@ def build(self):
 				cell_id_df = pd.DataFrame(obs_df['cell_id'][:1]).reset_index(drop=True)
 				X_df = pd.concat([cell_id_df, X_df], axis=1)
 				X_df.columns = ['cell_id'] + list(X_df.columns[1:])
-				chunk_size = 5000 
+				chunk_size = self.chunk_size 
 				print(f"Starting backed mode X table data insert. Total rows: {self.adata.shape[0]}")
 				for start in range(0, self.adata.shape[0], chunk_size):
 					start_time = time.time()

src/AnnSQL/MakeDb.py

@@ -5,14 +5,22 @@
 import os 
 
 class MakeDb:
-	def __init__(self, adata=None, db_name=None, db_path="db/", create_all_indexes=False, create_basic_indexes=False, convenience_view=True, layers=["X", "obs", "var", "var_names", "obsm", "varm", "obsp", "uns"]):
+	def __init__(self, adata=None, 
+						db_name=None, 
+						db_path="db/", 
+						create_all_indexes=False, 
+						create_basic_indexes=False, 
+						convenience_view=True, 
+						chunk_size=5000,
+						layers=["X", "obs", "var", "var_names", "obsm", "varm", "obsp", "uns"]):
 		self.adata = adata
 		self.db_name = db_name
 		self.db_path = db_path
 		self.layers = layers
 		self.create_all_indexes = create_all_indexes
 		self.create_basic_indexes = create_basic_indexes
 		self.convenience_view = convenience_view
+		self.chunk_size = chunk_size
 		self.validate_params()
 		self.build_db()
 
@@ -35,5 +43,5 @@ def create_db(self):
 
 	def build_db(self):
 		self.create_db()
-		BuildDb(adata=self.adata, conn=self.conn, create_all_indexes=self.create_all_indexes, create_basic_indexes=self.create_basic_indexes, convenience_view=self.convenience_view, layers=self.layers)
+		BuildDb(adata=self.adata, conn=self.conn, create_all_indexes=self.create_all_indexes, create_basic_indexes=self.create_basic_indexes, convenience_view=self.convenience_view, layers=self.layers, chunk_size=self.chunk_size)
 		self.conn.close()