Improved collect script with better filtering
This commit is contained in:
Niels Geens
117
claude.sh
117
claude.sh
@@ -1,117 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Initialize variables
|
||||
output_file="claude.txt"
|
||||
current_filter=""
|
||||
directories=()
|
||||
|
||||
# Parse command line arguments
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
-f|--filter)
|
||||
if [ -n "$2" ]; then
|
||||
current_filter="$2"
|
||||
shift 2
|
||||
else
|
||||
echo "Error: No filter pattern specified after -f/--filter" >&2
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
# Check if the argument is a valid directory
|
||||
if [ -d "$1" ]; then
|
||||
# Store directory with its associated filter
|
||||
directories+=("$1")
|
||||
directories+=("$current_filter")
|
||||
shift 1
|
||||
else
|
||||
echo "Error: Directory '$1' does not exist" >&2
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Clear/create output file
|
||||
> "$output_file"
|
||||
|
||||
# Add header
|
||||
echo "Directory Tree:" > "$output_file"
|
||||
echo "=============" >> "$output_file"
|
||||
|
||||
# Process each directory
|
||||
for ((i=0; i<${#directories[@]}; i+=2)); do
|
||||
dir="${directories[i]}"
|
||||
filter="${directories[i+1]}"
|
||||
|
||||
echo "Processing directory: $dir (filter: ${filter:-none})"
|
||||
|
||||
# Add directory to output
|
||||
echo -e "\n=== Directory: $dir ===" >> "$output_file"
|
||||
|
||||
# Change to target directory
|
||||
original_dir=$(pwd)
|
||||
cd "$dir" || continue
|
||||
|
||||
# Generate directory tree for this directory
|
||||
if command -v tree &> /dev/null; then
|
||||
if [ -f ".gitignore" ]; then
|
||||
tree -I "$(git check-ignore * .*)" >> "$output_file"
|
||||
else
|
||||
tree >> "$output_file"
|
||||
fi
|
||||
else
|
||||
find . -type f -name "*" | sort >> "$output_file"
|
||||
fi
|
||||
|
||||
# List files based on filter
|
||||
echo -e "\nFile Contents for $dir:" >> "$output_file"
|
||||
echo "=======================" >> "$output_file"
|
||||
|
||||
# Get list of files to process
|
||||
files_to_process=()
|
||||
|
||||
if [ -z "$filter" ]; then
|
||||
# No filter, use git ls-files if git repo, otherwise use find
|
||||
if git rev-parse --is-inside-work-tree &> /dev/null; then
|
||||
mapfile -d $'\0' files_to_process < <(git ls-files -z --cached --others --exclude-standard)
|
||||
else
|
||||
mapfile -d $'\0' files_to_process < <(find . -type f -not -path "*/\.*" -print0)
|
||||
fi
|
||||
else
|
||||
# Use filter
|
||||
if git rev-parse --is-inside-work-tree &> /dev/null; then
|
||||
mapfile -d $'\0' files_to_process < <(git ls-files -z --cached --others --exclude-standard | grep -z "$filter$")
|
||||
else
|
||||
mapfile -d $'\0' files_to_process < <(find . -type f -name "*$filter" -print0)
|
||||
fi
|
||||
fi
|
||||
|
||||
# Process each file
|
||||
for file in "${files_to_process[@]}"; do
|
||||
# Skip the output file itself
|
||||
if [ "$file" = "$output_file" ] || [ "$file" = "./$output_file" ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
# Skip non-existent files
|
||||
if [ ! -f "$file" ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
# Skip binary files
|
||||
if file "$file" | grep -q "binary"; then
|
||||
echo "Skipping binary file: $file" >> "$output_file"
|
||||
continue
|
||||
fi
|
||||
|
||||
echo -e "\n=== File: $dir/$file ===" >> "$output_file"
|
||||
echo -e "------------------------" >> "$output_file"
|
||||
cat "$file" >> "$output_file"
|
||||
done
|
||||
|
||||
# Return to original directory
|
||||
cd "$original_dir" || exit 1
|
||||
done
|
||||
|
||||
echo "Concatenation complete. Output written to $output_file"
|
||||
294
collect.sh
Normal file
294
collect.sh
Normal file
@@ -0,0 +1,294 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
declare -A FILTER_SETS=(
|
||||
["py"]="-f .*(\\.py|requirements.txt)$"
|
||||
["web"]="-f .*\\.(js|jsx|json|css|html)$"
|
||||
["doc"]="-f .*\\.md$"
|
||||
["deploy"]="-f .*(Dockerfile|\\.sh|\\.xsd)$"
|
||||
|
||||
["core"]="-s py ./sia ./tools -s deploy . -f ^(?!procedures/).*\\.md$ ."
|
||||
["webui"]="-s web ./web"
|
||||
["tests"]="-s py ./test"
|
||||
["procedures"]="-s doc ./procedures"
|
||||
)
|
||||
|
||||
OUTPUT="/dev/stdout"
|
||||
VERBOSE=0
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 [OPTIONS] [DIRS...]"
|
||||
echo
|
||||
echo "Options:"
|
||||
echo " -f, --filter PATTERN Regex filter for subsequent directories"
|
||||
echo " -s, --set SETNAME Use predefined set of parameters"
|
||||
echo " -o, --output FILE Output file (default: stdout)"
|
||||
echo " -v, -vv Verbose output"
|
||||
echo
|
||||
echo "Predefined Filter Sets:"
|
||||
for setname in $(echo "${!FILTER_SETS[@]}" | tr ' ' '\n' | sort); do
|
||||
printf " %-15s %s\n" "$setname:" "${FILTER_SETS[$setname]}"
|
||||
done
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Expand any set arguments to their full form
|
||||
expand_args() {
|
||||
local -a expanded=()
|
||||
local i=1
|
||||
local has_sets=0
|
||||
|
||||
# First pass: expand sets
|
||||
while [ $i -le $# ]; do
|
||||
local arg="${!i}"
|
||||
|
||||
case "$arg" in
|
||||
-s|--set)
|
||||
has_sets=1
|
||||
i=$((i+1))
|
||||
[ $i -le $# ] || { echo "Error: No set name specified"; usage; }
|
||||
local set_name="${!i}"
|
||||
|
||||
if [[ -v "FILTER_SETS[$set_name]" ]]; then
|
||||
# Add the expanded set parameters
|
||||
local -a set_args
|
||||
read -ra set_args <<< "${FILTER_SETS[$set_name]}"
|
||||
for set_arg in "${set_args[@]}"; do
|
||||
expanded+=("$set_arg")
|
||||
done
|
||||
else
|
||||
echo "Error: Unknown set '$set_name'"
|
||||
usage
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
# Preserve other arguments
|
||||
expanded+=("$arg")
|
||||
;;
|
||||
esac
|
||||
|
||||
i=$((i+1))
|
||||
done
|
||||
|
||||
# If we expanded sets, recursively expand again until no more sets
|
||||
if [ $has_sets -eq 1 ]; then
|
||||
expand_args "${expanded[@]}"
|
||||
else
|
||||
# No more sets, return the fully expanded arguments
|
||||
echo "${expanded[@]+"${expanded[@]}"}"
|
||||
fi
|
||||
}
|
||||
|
||||
# Process the final expanded command line
|
||||
process_command() {
|
||||
local filter=""
|
||||
declare -A dir_filters_map # Maps directory to array of filters
|
||||
|
||||
if ((VERBOSE >= 1)); then
|
||||
echo "Expanded arguments:"
|
||||
for arg in "$@"; do
|
||||
echo " $arg"
|
||||
done
|
||||
fi
|
||||
|
||||
# Process final arguments
|
||||
local i=1
|
||||
while [ $i -le $# ]; do
|
||||
local arg="${!i}"
|
||||
|
||||
case "$arg" in
|
||||
-f|--filter)
|
||||
i=$((i+1))
|
||||
[ $i -le $# ] || break
|
||||
filter="${!i}"
|
||||
;;
|
||||
-o|--output)
|
||||
i=$((i+1))
|
||||
[ $i -le $# ] || break
|
||||
OUTPUT="${!i}"
|
||||
;;
|
||||
-v)
|
||||
((VERBOSE++))
|
||||
;;
|
||||
-vv)
|
||||
VERBOSE=2
|
||||
;;
|
||||
-s|--set)
|
||||
echo "Warning: Set argument found after expansion"
|
||||
i=$((i+1))
|
||||
;;
|
||||
-*)
|
||||
# Skip other options
|
||||
;;
|
||||
*)
|
||||
# It's a directory
|
||||
if [ -e "$arg" ]; then
|
||||
# Add filter to this directory's filter list
|
||||
if [ -n "$filter" ]; then
|
||||
if [ -n "${dir_filters_map[$arg]:-}" ]; then
|
||||
dir_filters_map["$arg"]="${dir_filters_map["$arg"]}|$filter"
|
||||
else
|
||||
dir_filters_map["$arg"]="$filter"
|
||||
fi
|
||||
if ((VERBOSE >= 1)); then
|
||||
echo "Adding filter: $filter to directory: $arg"
|
||||
fi
|
||||
else
|
||||
# If no filter specified, ensure directory is in the map
|
||||
[ -z "${dir_filters_map[$arg]:-}" ] && dir_filters_map["$arg"]=""
|
||||
if ((VERBOSE >= 1)); then
|
||||
echo "Adding directory with no filter: $arg"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
echo "Warning: Directory '$arg' not found"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
|
||||
i=$((i+1))
|
||||
done
|
||||
|
||||
# Convert output path and check conflicts
|
||||
output_abs_path=$(realpath -m "$OUTPUT")
|
||||
output_rel_path=$(realpath --relative-to=. "$output_abs_path")
|
||||
|
||||
# Generate directory tree
|
||||
{
|
||||
echo "Directory Tree:"
|
||||
if command -v tree &>/dev/null; then
|
||||
tree -a -I '.git' .
|
||||
else
|
||||
find . -name .git -prune -o -print | sed -e 's;[^/]*/;|____;g;s;____|; |;g'
|
||||
fi
|
||||
} > "$OUTPUT"
|
||||
|
||||
# Process directories and collect files
|
||||
declare -a all_files=()
|
||||
|
||||
# Cache git-ignored files once
|
||||
declare -a git_ignored_files=()
|
||||
if git rev-parse --is-inside-work-tree &>/dev/null; then
|
||||
while IFS= read -r ignored_file; do
|
||||
git_ignored_files+=("$ignored_file")
|
||||
done < <(git ls-files --others --ignored --exclude-standard 2>/dev/null)
|
||||
|
||||
if ((VERBOSE >= 1)); then
|
||||
echo "Cached ${#git_ignored_files[@]} git-ignored files"
|
||||
fi
|
||||
fi
|
||||
|
||||
for dir in "${!dir_filters_map[@]}"; do
|
||||
local filters="${dir_filters_map[$dir]}"
|
||||
|
||||
if ((VERBOSE >= 1)); then
|
||||
echo "Processing directory: $dir with filters: ${filters:-none}"
|
||||
fi
|
||||
|
||||
# Get all files in the directory first
|
||||
declare -a dir_files=()
|
||||
while IFS= read -r -d $'\0' file; do
|
||||
local rel_path="${file#./}"
|
||||
[[ -z "$rel_path" || "$rel_path" == "." ]] && continue
|
||||
|
||||
# Skip output file
|
||||
[[ "$rel_path" == "$output_rel_path" ]] && continue
|
||||
|
||||
# Check if file is git-ignored using cached list
|
||||
local is_ignored=0
|
||||
for ignored in "${git_ignored_files[@]}"; do
|
||||
if [[ "$rel_path" == "$ignored" ]]; then
|
||||
is_ignored=1
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
# Skip git-ignored files
|
||||
[[ $is_ignored -eq 1 ]] && continue
|
||||
|
||||
dir_files+=("$rel_path")
|
||||
done < <(find "$dir" \( -path '*/.git' -prune \) -o \( -path "./$output_rel_path" -prune \) -o -type f -print0 2>/dev/null)
|
||||
|
||||
if ((VERBOSE >= 1)); then
|
||||
echo " Found ${#dir_files[@]} files in directory"
|
||||
fi
|
||||
|
||||
# If filters are specified, apply them to the full file list
|
||||
if [[ -n "$filters" ]]; then
|
||||
# Create temp file with all paths
|
||||
local temp_file=$(mktemp)
|
||||
printf "%s\n" "${dir_files[@]}" > "$temp_file"
|
||||
|
||||
# Apply filters to get matching files
|
||||
declare -a matching_files=()
|
||||
while IFS= read -r matched; do
|
||||
[[ -n "$matched" ]] && matching_files+=("$matched")
|
||||
(( VERBOSE >= 2 )) && echo " Included: $matched"
|
||||
done < <(grep -E "($filters)" "$temp_file")
|
||||
|
||||
# Add matching files to all_files
|
||||
all_files+=("${matching_files[@]}")
|
||||
|
||||
# Clean up
|
||||
rm -f "$temp_file"
|
||||
|
||||
if ((VERBOSE >= 1)); then
|
||||
echo " Matched ${#matching_files[@]} files after filtering"
|
||||
fi
|
||||
else
|
||||
# No filter, add all files
|
||||
all_files+=("${dir_files[@]}")
|
||||
|
||||
if ((VERBOSE >= 2)); then
|
||||
for file in "${dir_files[@]}"; do
|
||||
echo " Included: $file"
|
||||
done
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# Generate unique file list
|
||||
if [ ${#all_files[@]} -gt 0 ]; then
|
||||
readarray -t unique_files < <(printf "%s\n" "${all_files[@]}" | sort -u)
|
||||
|
||||
# Append file contents to output
|
||||
{
|
||||
echo -e "\n\nFile Contents:"
|
||||
for file in "${unique_files[@]}"; do
|
||||
if [[ -L "$file" ]]; then
|
||||
target=$(readlink -f "$file" || echo "unknown")
|
||||
echo -e "\n===== SYMLINK: $file → $target ====="
|
||||
else
|
||||
echo -e "\n===== FILE: $file ====="
|
||||
cat "$file" 2>/dev/null || echo "Error: Unable to read file"
|
||||
fi
|
||||
done
|
||||
} >> "$OUTPUT"
|
||||
fi
|
||||
}
|
||||
|
||||
main() {
|
||||
# No arguments? Show usage
|
||||
[ $# -eq 0 ] && usage
|
||||
|
||||
# First scan for verbose flags
|
||||
for arg in "$@"; do
|
||||
if [[ "$arg" == "-v" ]]; then
|
||||
((VERBOSE++))
|
||||
elif [[ "$arg" == "-vv" ]]; then
|
||||
VERBOSE=2
|
||||
fi
|
||||
done
|
||||
|
||||
# Expand all set arguments recursively
|
||||
local -a expanded_args
|
||||
read -ra expanded_args <<< "$(expand_args "$@")"
|
||||
|
||||
# Process the completely expanded arguments
|
||||
process_command "${expanded_args[@]}"
|
||||
|
||||
echo "Concatenation complete. Output written to $OUTPUT" >&2
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user