I Inherited 47,000 Lines of Terraform Spaghetti — Here's How I Untangled It Without Burning Production

# main.tf — 8,400 lines# "Managed" networking, compute, databases, DNS, IAM, monitoring,# and somehow... a CloudFront distribution for a marketing site# that was decommissioned in 2023.resource"aws_vpc""main"{...}resource"aws_instance""api_server_1"{...}resource"aws_instance""api_server_2"{...}# ... 200 more instances ...resource"aws_rds_instance""prod_db"{...}resource"aws_iam_role""god_mode"{...}# yes, really

# First, I mapped resource dependencies visually
terraform graph | dot -Tsvg> infra-dependency-map.svg
# Then, split by domain boundaries
terraform state mv'aws_vpc.main'-state-out=networking/terraform.tfstate
terraform state mv'aws_subnet.public[0]'-state-out=networking/terraform.tfstate
terraform state mv'aws_subnet.public[1]'-state-out=networking/terraform.tfstate

# In compute/main.tfdata"terraform_remote_state""networking"{backend="s3"config={bucket="company-terraform-state"key="foundation/terraform.tfstate"region="us-east-1"}}resource"aws_ecs_service""api"{# Reference networking outputs safelynetwork_configuration{subnets=data.terraform_remote_state.networking.outputs.private_subnet_ids}}

environments/
├── dev/
│ └── main.tf # 1,200 lines
├── staging/
│ └── main.tf # 1,200 lines (95% identical to dev)
├── prod/
│ └── main.tf # 1,200 lines (90% identical... with 47 "hotfixes")
└── dr/
└── main.tf # 1,200 lines (copied from prod 8 months ago, never updated)

# modules/api-platform/main.tfvariable"environment"{type=stringvalidation{condition=contains(["dev","staging","prod","dr"],var.environment)error_message="Environment must be dev, staging, prod, or dr."}}variable"config"{type=object({instance_type=stringmin_capacity=numbermax_capacity=numberenable_waf=boolmulti_az=boolbackup_retention=number})}locals{# Environment-specific defaults that document WHY they differenv_config={dev={instance_type="t3.medium"min_capacity=1max_capacity=2enable_waf=falsemulti_az=falsebackup_retention=1}prod={instance_type="m5.xlarge"min_capacity=3max_capacity=20enable_waf=truemulti_az=truebackup_retention=35}}}

deploy_prod:stage:deployscript:-terraform init-terraform apply -auto-approve# 🚨 WHATonly:-main

fix: revert the revert of the fix
fix: actually fix prod this time
fix: ok THIS one fixes it
revert: revert everything from today

# .github/workflows/terraform.ymlname:"Terraform"on:pull_request:paths:['infrastructure/**']push:branches:[main]paths:['infrastructure/**']jobs:plan:runs-on:ubuntu-lateststeps:-uses:actions/checkout@v4-name:Terraform Planid:planrun:|terraform initterraform plan -no-color -out=tfplan \-detailed-exitcode 2>&1 | tee plan_output.txtcontinue-on-error:true-name:Comment Plan on PRuses:actions/github-script@v7if:github.event_name == 'pull_request'with:script:|const fs = require('fs');const plan = fs.readFileSync('plan_output.txt', 'utf8');const truncated = plan.length > 60000 ? plan.substring(0, 60000) + '\n\n... truncated ...' : plan;github.rest.issues.createComment({issue_number: context.issue.number,owner: context.repo.owner,repo: context.repo.repo,body: `## Terraform Plan Output\n\`\`\`\n${truncated}\n\`\`\``});-name:Upload Plan Artifactuses:actions/upload-artifact@v4with:name:tfplanpath:tfplanapply:needs:planruns-on:ubuntu-latestif:github.ref == 'refs/heads/main' && github.event_name == 'push'environment:production# Requires manual approvalsteps:-uses:actions/checkout@v4-name:Download Planuses:actions/download-artifact@v4with:name:tfplan-name:Terraform Applyrun:terraform apply tfplan# Apply ONLY the reviewed plan

resource"aws_db_instance""prod"{engine="postgres"instance_class="db.r5.2xlarge"username="admin"password="Pr0d_P@ssw0rd_2022!"# I wish I was jokingpublicly_accessible=true# I really wish I was joking}

# Use a data source to pull secrets at plan/apply timedata"aws_secretsmanager_secret_version""db_password"{secret_id="prod/rds/master-password"}resource"aws_db_instance""prod"{engine="postgres"instance_class="db.r5.2xlarge"username="admin"password=data.aws_secretsmanager_secret_version.db_password.secret_stringpublicly_accessible=false# Prevent Terraform from detecting password "drift"lifecycle{ignore_changes=[password]}}

# backend.tfterraform{backend"s3"{bucket="company-terraform-state"key="prod/data/terraform.tfstate"region="us-east-1"encrypt=true# SSE-KMS encryptionkms_key_id="arn:aws:kms:us-east-1:xxx:key/yyy"dynamodb_table="terraform-state-lock"}}

resource"aws_ecs_task_definition""api"{family="api"network_mode="awsvpc"requires_compatibilities=["FARGATE"]cpu=1024memory=2048execution_role_arn=aws_iam_role.ecs_execution.arntask_role_arn=aws_iam_role.ecs_task.arncontainer_definitions=jsonencode([{name="api"image="company/api:latest"# 🚨 LATEST TAG IN PRODportMappings=[{containerPort=8080}]environment=[{name="DB_HOST",value="prod-db.cluster-xxx.us-east-1.rds.amazonaws.com"},{name="DB_NAME",value="production"},{name="REDIS_URL",value="prod-redis.xxx.cache.amazonaws.com:6379"},# ... 45 more environment variables hardcoded here ...]logConfiguration={logDriver="awslogs"options={"awslogs-group"="/ecs/api""awslogs-region"="us-east-1""awslogs-stream-prefix"="api"}}# ... 80 more lines of health checks, mount points, ulimits ...}])}

# Use templatefile for complex JSON — it's testable and readableresource"aws_ecs_task_definition""api"{family="api-${var.environment}"network_mode="awsvpc"requires_compatibilities=["FARGATE"]cpu=var.task_cpumemory=var.task_memoryexecution_role_arn=aws_iam_role.ecs_execution.arntask_role_arn=aws_iam_role.ecs_task.arncontainer_definitions=templatefile("${path.module}/templates/api-container.json.tpl",{image_tag=var.image_tag# Pinned, passed from CI/CDenvironment=var.environmentdb_host=data.aws_ssm_parameter.db_host.valueredis_url=data.aws_ssm_parameter.redis_url.valuelog_group=aws_cloudwatch_log_group.api.nameaws_region=data.aws_region.current.name})}

# 1. Enable state file encryption and locking NOW# 2. Add branch protection — no direct pushes to main# 3. Run terraform plan and SAVE the output as your baseline
terraform plan -no-color> baseline_plan_$(date +%Y%m%d).txt
# 4. Enable detailed audit logging on your state bucket

# Use terraform state list to inventory everything
terraform state list > all_resources.txt
wc-l all_resources.txt # Mine had 2,847 resources# Group by service domaingrep"aws_vpc\|aws_subnet\|aws_route" all_resources.txt > networking.txt
grep"aws_iam\|aws_kms" all_resources.txt > security.txt
grep"aws_rds\|aws_elasticache\|aws_s3" all_resources.txt > data.txt
grep"aws_ecs\|aws_alb\|aws_autoscaling" all_resources.txt > compute.txt

# .github/workflows/drift-detection.ymlname:"DriftDetection"on:schedule:-cron:'06**1-5'# Every weekday at 6 AMjobs:detect-drift:runs-on:ubuntu-lateststrategy:matrix:layer:[foundation,security,data,compute,edge]steps:-uses:actions/checkout@v4-name:Terraform Plan (Drift Check)id:planworking-directory:infrastructure/${{ matrix.layer }}run:|terraform initterraform plan -detailed-exitcode -no-color > plan.txt 2>&1echo "exitcode=$?" >> $GITHUB_OUTPUTcontinue-on-error:true-name:Alert on Driftif:steps.plan.outputs.exitcode == '2'run:|# Exit code 2 = changes detected (drift!)curl -X POST "${{ secrets.SLACK_WEBHOOK }}" \-H 'Content-type: application/json' \-d "{\"text\":\"🚨 Drift detected in *${{ matrix.layer }}* layer. Check the plan output.\"}"