Improved collect script with better filtering

claude.sh

@@ -1,117 +0,0 @@
-#!/bin/bash
-
-# Initialize variables
-output_file="claude.txt"
-current_filter=""
-directories=()
-
-# Parse command line arguments
-while [[ $# -gt 0 ]]; do
-    case $1 in
-        -f|--filter)
-            if [ -n "$2" ]; then
-                current_filter="$2"
-                shift 2
-            else
-                echo "Error: No filter pattern specified after -f/--filter" >&2
-                exit 1
-            fi
-            ;;
-        *)
-            # Check if the argument is a valid directory
-            if [ -d "$1" ]; then
-                # Store directory with its associated filter
-                directories+=("$1")
-                directories+=("$current_filter")
-                shift 1
-            else
-                echo "Error: Directory '$1' does not exist" >&2
-                exit 1
-            fi
-            ;;
-    esac
-done
-
-# Clear/create output file
-> "$output_file"
-
-# Add header
-echo "Directory Tree:" > "$output_file"
-echo "=============" >> "$output_file"
-
-# Process each directory
-for ((i=0; i<${#directories[@]}; i+=2)); do
-    dir="${directories[i]}"
-    filter="${directories[i+1]}"
-    
-    echo "Processing directory: $dir (filter: ${filter:-none})"
-    
-    # Add directory to output
-    echo -e "\n=== Directory: $dir ===" >> "$output_file"
-    
-    # Change to target directory
-    original_dir=$(pwd)
-    cd "$dir" || continue
-    
-    # Generate directory tree for this directory
-    if command -v tree &> /dev/null; then
-        if [ -f ".gitignore" ]; then
-            tree -I "$(git check-ignore * .*)" >> "$output_file"
-        else
-            tree >> "$output_file"
-        fi
-    else
-        find . -type f -name "*" | sort >> "$output_file"
-    fi
-    
-    # List files based on filter
-    echo -e "\nFile Contents for $dir:" >> "$output_file"
-    echo "=======================" >> "$output_file"
-    
-    # Get list of files to process
-    files_to_process=()
-    
-    if [ -z "$filter" ]; then
-        # No filter, use git ls-files if git repo, otherwise use find
-        if git rev-parse --is-inside-work-tree &> /dev/null; then
-            mapfile -d $'\0' files_to_process < <(git ls-files -z --cached --others --exclude-standard)
-        else
-            mapfile -d $'\0' files_to_process < <(find . -type f -not -path "*/\.*" -print0)
-        fi
-    else
-        # Use filter
-        if git rev-parse --is-inside-work-tree &> /dev/null; then
-            mapfile -d $'\0' files_to_process < <(git ls-files -z --cached --others --exclude-standard | grep -z "$filter$")
-        else
-            mapfile -d $'\0' files_to_process < <(find . -type f -name "*$filter" -print0)
-        fi
-    fi
-    
-    # Process each file
-    for file in "${files_to_process[@]}"; do
-        # Skip the output file itself
-        if [ "$file" = "$output_file" ] || [ "$file" = "./$output_file" ]; then
-            continue
-        fi
-        
-        # Skip non-existent files
-        if [ ! -f "$file" ]; then
-            continue
-        fi
-        
-        # Skip binary files
-        if file "$file" | grep -q "binary"; then
-            echo "Skipping binary file: $file" >> "$output_file"
-            continue
-        fi
-        
-        echo -e "\n=== File: $dir/$file ===" >> "$output_file"
-        echo -e "------------------------" >> "$output_file"
-        cat "$file" >> "$output_file"
-    done
-    
-    # Return to original directory
-    cd "$original_dir" || exit 1
-done
-
-echo "Concatenation complete. Output written to $output_file"

collect.sh

@@ -0,0 +1,294 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+declare -A FILTER_SETS=(
+  ["py"]="-f .*(\\.py|requirements.txt)$"
+  ["web"]="-f .*\\.(js|jsx|json|css|html)$"
+  ["doc"]="-f .*\\.md$"
+  ["deploy"]="-f .*(Dockerfile|\\.sh|\\.xsd)$"
+  
+  ["core"]="-s py ./sia ./tools -s deploy . -f ^(?!procedures/).*\\.md$ ."
+  ["webui"]="-s web ./web"
+  ["tests"]="-s py ./test"
+  ["procedures"]="-s doc ./procedures"
+)
+
+OUTPUT="/dev/stdout"
+VERBOSE=0
+
+usage() {
+  echo "Usage: $0 [OPTIONS] [DIRS...]"
+  echo
+  echo "Options:"
+  echo "  -f, --filter PATTERN   Regex filter for subsequent directories"
+  echo "  -s, --set SETNAME      Use predefined set of parameters"
+  echo "  -o, --output FILE      Output file (default: stdout)"
+  echo "  -v, -vv                Verbose output"
+  echo
+  echo "Predefined Filter Sets:"
+  for setname in $(echo "${!FILTER_SETS[@]}" | tr ' ' '\n' | sort); do
+    printf "  %-15s %s\n" "$setname:" "${FILTER_SETS[$setname]}"
+  done
+  exit 1
+}
+
+# Expand any set arguments to their full form
+expand_args() {
+  local -a expanded=()
+  local i=1
+  local has_sets=0
+  
+  # First pass: expand sets
+  while [ $i -le $# ]; do
+    local arg="${!i}"
+    
+    case "$arg" in
+      -s|--set)
+        has_sets=1
+        i=$((i+1))
+        [ $i -le $# ] || { echo "Error: No set name specified"; usage; }
+        local set_name="${!i}"
+        
+        if [[ -v "FILTER_SETS[$set_name]" ]]; then
+          # Add the expanded set parameters
+          local -a set_args
+          read -ra set_args <<< "${FILTER_SETS[$set_name]}"
+          for set_arg in "${set_args[@]}"; do
+            expanded+=("$set_arg")
+          done
+        else
+          echo "Error: Unknown set '$set_name'"
+          usage
+        fi
+        ;;
+      *)
+        # Preserve other arguments
+        expanded+=("$arg")
+        ;;
+    esac
+    
+    i=$((i+1))
+  done
+  
+  # If we expanded sets, recursively expand again until no more sets
+  if [ $has_sets -eq 1 ]; then
+    expand_args "${expanded[@]}"
+  else
+    # No more sets, return the fully expanded arguments
+    echo "${expanded[@]+"${expanded[@]}"}"
+  fi
+}
+
+# Process the final expanded command line
+process_command() {
+  local filter=""
+  declare -A dir_filters_map  # Maps directory to array of filters
+  
+  if ((VERBOSE >= 1)); then
+    echo "Expanded arguments:"
+    for arg in "$@"; do
+      echo "  $arg"
+    done
+  fi
+  
+  # Process final arguments
+  local i=1
+  while [ $i -le $# ]; do
+    local arg="${!i}"
+    
+    case "$arg" in
+      -f|--filter)
+        i=$((i+1))
+        [ $i -le $# ] || break
+        filter="${!i}"
+        ;;
+      -o|--output)
+        i=$((i+1))
+        [ $i -le $# ] || break
+        OUTPUT="${!i}"
+        ;;
+      -v)
+        ((VERBOSE++))
+        ;;
+      -vv)
+        VERBOSE=2
+        ;;
+      -s|--set)
+        echo "Warning: Set argument found after expansion"
+        i=$((i+1))
+        ;;
+      -*)
+        # Skip other options
+        ;;
+      *)
+        # It's a directory
+        if [ -e "$arg" ]; then
+          # Add filter to this directory's filter list
+          if [ -n "$filter" ]; then
+            if [ -n "${dir_filters_map[$arg]:-}" ]; then
+              dir_filters_map["$arg"]="${dir_filters_map["$arg"]}|$filter"
+            else
+              dir_filters_map["$arg"]="$filter"
+            fi
+            if ((VERBOSE >= 1)); then
+              echo "Adding filter: $filter to directory: $arg"
+            fi
+          else
+            # If no filter specified, ensure directory is in the map
+            [ -z "${dir_filters_map[$arg]:-}" ] && dir_filters_map["$arg"]=""
+            if ((VERBOSE >= 1)); then
+              echo "Adding directory with no filter: $arg"
+            fi
+          fi
+        else
+          echo "Warning: Directory '$arg' not found"
+        fi
+        ;;
+    esac
+    
+    i=$((i+1))
+  done
+  
+  # Convert output path and check conflicts
+  output_abs_path=$(realpath -m "$OUTPUT")
+  output_rel_path=$(realpath --relative-to=. "$output_abs_path")
+  
+  # Generate directory tree
+  {
+    echo "Directory Tree:"
+    if command -v tree &>/dev/null; then
+      tree -a -I '.git' .
+    else
+      find . -name .git -prune -o -print | sed -e 's;[^/]*/;|____;g;s;____|; |;g'
+    fi
+  } > "$OUTPUT"
+  
+  # Process directories and collect files
+  declare -a all_files=()
+  
+  # Cache git-ignored files once
+  declare -a git_ignored_files=()
+  if git rev-parse --is-inside-work-tree &>/dev/null; then
+    while IFS= read -r ignored_file; do
+      git_ignored_files+=("$ignored_file")
+    done < <(git ls-files --others --ignored --exclude-standard 2>/dev/null)
+    
+    if ((VERBOSE >= 1)); then
+      echo "Cached ${#git_ignored_files[@]} git-ignored files"
+    fi
+  fi
+  
+  for dir in "${!dir_filters_map[@]}"; do
+    local filters="${dir_filters_map[$dir]}"
+    
+    if ((VERBOSE >= 1)); then
+      echo "Processing directory: $dir with filters: ${filters:-none}"
+    fi
+    
+    # Get all files in the directory first
+    declare -a dir_files=()
+    while IFS= read -r -d $'\0' file; do
+      local rel_path="${file#./}"
+      [[ -z "$rel_path" || "$rel_path" == "." ]] && continue
+      
+      # Skip output file
+      [[ "$rel_path" == "$output_rel_path" ]] && continue
+      
+      # Check if file is git-ignored using cached list
+      local is_ignored=0
+      for ignored in "${git_ignored_files[@]}"; do
+        if [[ "$rel_path" == "$ignored" ]]; then
+          is_ignored=1
+          break
+        fi
+      done
+      
+      # Skip git-ignored files
+      [[ $is_ignored -eq 1 ]] && continue
+      
+      dir_files+=("$rel_path")
+    done < <(find "$dir" \( -path '*/.git' -prune \) -o \( -path "./$output_rel_path" -prune \) -o -type f -print0 2>/dev/null)
+    
+    if ((VERBOSE >= 1)); then
+      echo "  Found ${#dir_files[@]} files in directory"
+    fi
+    
+    # If filters are specified, apply them to the full file list
+    if [[ -n "$filters" ]]; then
+      # Create temp file with all paths
+      local temp_file=$(mktemp)
+      printf "%s\n" "${dir_files[@]}" > "$temp_file"
+      
+      # Apply filters to get matching files
+      declare -a matching_files=()
+      while IFS= read -r matched; do
+        [[ -n "$matched" ]] && matching_files+=("$matched")
+        (( VERBOSE >= 2 )) && echo "  Included: $matched"
+      done < <(grep -E "($filters)" "$temp_file")
+      
+      # Add matching files to all_files
+      all_files+=("${matching_files[@]}")
+      
+      # Clean up
+      rm -f "$temp_file"
+      
+      if ((VERBOSE >= 1)); then
+        echo "  Matched ${#matching_files[@]} files after filtering"
+      fi
+    else
+      # No filter, add all files
+      all_files+=("${dir_files[@]}")
+      
+      if ((VERBOSE >= 2)); then
+        for file in "${dir_files[@]}"; do
+          echo "  Included: $file"
+        done
+      fi
+    fi
+  done
+  
+  # Generate unique file list
+  if [ ${#all_files[@]} -gt 0 ]; then
+    readarray -t unique_files < <(printf "%s\n" "${all_files[@]}" | sort -u)
+    
+    # Append file contents to output
+    {
+      echo -e "\n\nFile Contents:"
+      for file in "${unique_files[@]}"; do
+        if [[ -L "$file" ]]; then
+          target=$(readlink -f "$file" || echo "unknown")
+          echo -e "\n===== SYMLINK: $file → $target ====="
+        else
+          echo -e "\n===== FILE: $file ====="
+          cat "$file" 2>/dev/null || echo "Error: Unable to read file"
+        fi
+      done
+    } >> "$OUTPUT"
+  fi
+}
+
+main() {
+  # No arguments? Show usage
+  [ $# -eq 0 ] && usage
+  
+  # First scan for verbose flags
+  for arg in "$@"; do
+    if [[ "$arg" == "-v" ]]; then
+      ((VERBOSE++))
+    elif [[ "$arg" == "-vv" ]]; then
+      VERBOSE=2
+    fi
+  done
+  
+  # Expand all set arguments recursively
+  local -a expanded_args
+  read -ra expanded_args <<< "$(expand_args "$@")"
+  
+  # Process the completely expanded arguments
+  process_command "${expanded_args[@]}"
+  
+  echo "Concatenation complete. Output written to $OUTPUT" >&2
+}
+
+main "$@"