Merge pull request #71 from tomarv2/develop

Pulling refs/heads/develop into main

Author: tomarv2

README.md

@@ -26,7 +26,7 @@
 ## Versions
 
 - Module tested for Terraform 1.0.1.
-- `databricks/databricks` provider version [0.5.7](https://registry.terraform.io/providers/databricks/databricks/latest)
+- `databricks/databricks` provider version [1.3.0](https://registry.terraform.io/providers/databricks/databricks/latest)
 - AWS provider version [4.14](https://registry.terraform.io/providers/hashicorp/aws/latest).
 - `main` branch: Provider versions not pinned to keep up with Terraform releases.
 - `tags` releases: Tags are pinned with versions (use <a href="https://github.com/tomarv2/terraform-databricks-workspace-management/tags" alt="GitHub tag">

examples/jobs/4_job_on_existing_cluster_with_new_notebooks/main.tf

@@ -12,16 +12,36 @@ terraform {
   }
 }
 
+module "cluster" {
+  source = "../../../"
+  # ------------------------------------------------
+  # CLUSTER
+  # ------------------------------------------------
+  deploy_cluster = true
+  fixed_value    = 0
+
+#  libraries = {
+#    maven = {
+#      "com.microsoft.azure:azure-eventhubs-spark_2.12:2.3.21" = {}
+#    }
+#    python_wheel = {}
+#  }
 
-module "databricks_workspace_management" {
+  # ------------------------------------------------
+  # Do not change the teamid, prjid once set.
+  teamid = var.teamid
+  prjid  = var.prjid
+}
+
+module "jobs" {
   source = "../../../"
 
   # ------------------------------------------------
   # JOB
   # ------------------------------------------------
   deploy_jobs               = true
-  cluster_id                = "0907-052446-bike152"
-  fixed_value               = 1
+  cluster_id                = module.cluster.cluster_id
+  #fixed_value               = 1
   retry_on_timeout          = false
   max_retries               = 3
   timeout                   = 30
@@ -57,6 +77,13 @@ module "databricks_workspace_management" {
       local_path = "notebooks/sample2.py"
     }
   ]
+  jobs_access_control = [
+    {
+      group_name       = "demo"
+      permission_level = "CAN_MANAGE_RUN"
+    }
+  ]
+  depends_on = [module.cluster]
   # ------------------------------------------------
   # Do not change the teamid, prjid once set.
   teamid = var.teamid

examples/jobs/4_job_on_existing_cluster_with_new_notebooks/outputs.tf

@@ -1,15 +1,15 @@
 output "notebook_url" {
   description = "databricks notebook url"
-  value       = module.databricks_workspace_management.notebook_url
+  value       = module.jobs.notebook_url
 }
 
 output "job_url" {
   description = "databricks job url"
-  value       = module.databricks_workspace_management.existing_cluster_new_job_new_notebooks_job
+  value       = module.jobs.existing_cluster_new_job_new_notebooks_job
 }
 
 output "job_id" {
   description = "databricks job id"
-  value       = module.databricks_workspace_management.existing_cluster_new_job_new_notebooks_id
+  value       = module.jobs.existing_cluster_new_job_new_notebooks_id
 }
 

job.tf

@@ -3,7 +3,7 @@
 # 1. NEW CLUSTER WITH NEW NOTEBOOKS
 # ------------------------------------------------
 resource "databricks_job" "new_cluster_new_job_new_notebooks" {
-  for_each = (var.deploy_jobs == true && var.cluster_id == null && var.deploy_cluster == true && var.local_notebooks != null) ? { for p in var.local_notebooks : "${p.job_name}-${p.local_path}" => p } : {}
+  for_each = (var.deploy_jobs == true && var.deploy_cluster == true && var.local_notebooks != null) ? { for p in var.local_notebooks : "${p.job_name}-${p.local_path}" => p } : {}
 
   name = "${each.value.job_name} (Terraform managed)"
 
@@ -93,7 +93,7 @@ resource "databricks_job" "new_cluster_new_job_new_notebooks" {
 # 2. NEW CLUSTER WITH EXITING NOTEBOOKS
 # ------------------------------------------------
 resource "databricks_job" "new_cluster_new_job_existing_notebooks" {
-  for_each = (var.deploy_jobs == true && var.cluster_id == null && var.deploy_cluster == true && var.remote_notebooks != null) ? { for p in var.remote_notebooks : "${p.job_name}-${p.path}" => p } : {}
+  for_each = (var.deploy_jobs == true && var.deploy_cluster == true && var.remote_notebooks != null) ? { for p in var.remote_notebooks : "${p.job_name}-${p.path}" => p } : {}
 
   name = "${each.value.job_name} (Terraform managed)"
 
@@ -184,7 +184,7 @@ resource "databricks_job" "new_cluster_new_job_existing_notebooks" {
 # 3. EXISTING CLUSTER WITH NEW NOTEBOOKS
 # ------------------------------------------------
 resource "databricks_job" "existing_cluster_new_job_new_notebooks" {
-  for_each = (var.deploy_jobs == true && (var.cluster_id != null || var.deploy_cluster == false) && var.local_notebooks != null) ? { for p in var.local_notebooks : "${p.job_name}-${p.local_path}" => p } : {}
+  for_each = (var.deploy_jobs == true && (var.deploy_cluster == false) && var.local_notebooks != null) ? { for p in var.local_notebooks : "${p.job_name}-${p.local_path}" => p } : {}
 
   name                = "${each.value.job_name} (Terraform managed)"
   existing_cluster_id = local.cluster_info
@@ -224,7 +224,7 @@ resource "databricks_job" "existing_cluster_new_job_new_notebooks" {
 # 4. EXISTING CLUSTER WITH EXITING NOTEBOOKS
 # ------------------------------------------------
 resource "databricks_job" "existing_cluster_new_job_existing_notebooks" {
-  for_each = var.deploy_jobs == true && (var.cluster_id != null || var.deploy_cluster == false) && var.remote_notebooks != null ? { for p in var.remote_notebooks : "${p.job_name}-${p.path}" => p } : {}
+  for_each = var.deploy_jobs == true && (var.deploy_cluster == false) && var.remote_notebooks != null ? { for p in var.remote_notebooks : "${p.job_name}-${p.path}" => p } : {}
 
   name                = "${each.value.job_name} (Terraform managed)"
   existing_cluster_id = local.cluster_info

permissions.tf

@@ -72,7 +72,7 @@ resource "databricks_permissions" "policy" {
 # 1. NEW CLUSTER WITH NEW NOTEBOOKS
 # ------------------------------------------------
 resource "databricks_permissions" "new_cluster_new_job_new_notebooks" {
-  for_each = (var.jobs_access_control != null && var.deploy_jobs == true && var.cluster_id == null && var.local_notebooks != null) ? databricks_job.new_cluster_new_job_new_notebooks : {}
+  for_each = (var.jobs_access_control != null && var.deploy_jobs == true && var.local_notebooks != null) ? databricks_job.new_cluster_new_job_new_notebooks : {}
 
   job_id = each.value.id
 
@@ -89,7 +89,7 @@ resource "databricks_permissions" "new_cluster_new_job_new_notebooks" {
 # 2. EXISTING CLUSTER WITH NEW NOTEBOOKS
 # ------------------------------------------------
 resource "databricks_permissions" "existing_cluster_new_job_new_notebooks" {
-  for_each = (var.jobs_access_control != null && var.deploy_jobs == true && var.cluster_id != null && var.local_notebooks != null) ? databricks_job.existing_cluster_new_job_new_notebooks : {}
+  for_each = (var.jobs_access_control != null && var.deploy_jobs == true && var.local_notebooks != null) ? databricks_job.existing_cluster_new_job_new_notebooks : {}
 
   job_id = each.value.id
 
@@ -106,7 +106,7 @@ resource "databricks_permissions" "existing_cluster_new_job_new_notebooks" {
 # 3. NEW CLUSTER WITH EXITING NOTEBOOKS
 # ------------------------------------------------
 resource "databricks_permissions" "new_cluster_new_job_existing_notebooks" {
-  for_each = (var.jobs_access_control != null && var.deploy_jobs == true && var.cluster_id == null && var.remote_notebooks != null) ? databricks_job.new_cluster_new_job_existing_notebooks : {}
+  for_each = (var.jobs_access_control != null && var.deploy_jobs == true && var.remote_notebooks != null) ? databricks_job.new_cluster_new_job_existing_notebooks : {}
 
   job_id = each.value.id
 
@@ -123,7 +123,7 @@ resource "databricks_permissions" "new_cluster_new_job_existing_notebooks" {
 # 4. EXISTING CLUSTER WITH EXITING NOTEBOOKS
 # ------------------------------------------------
 resource "databricks_permissions" "existing_cluster_new_job_existing_notebooks" {
-  for_each = (var.jobs_access_control != null && var.deploy_jobs == true && var.cluster_id != null && var.remote_notebooks != null) ? databricks_job.existing_cluster_new_job_existing_notebooks : {}
+  for_each = (var.jobs_access_control != null && var.deploy_jobs == true && var.remote_notebooks != null) ? databricks_job.existing_cluster_new_job_existing_notebooks : {}
 
   job_id = each.value.id