(no message) Changes: import_masterclass.py
import_masterclass.sh
This commit is contained in:
Steven (Zen)
parent
4c4cfd1277
commit
3d37235238
276
import_masterclass.py
Normal file
276
import_masterclass.py
Normal file
@ -0,0 +1,276 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
import_masterclass.py
|
||||
Creates a Masterclass.library in Eagle format from /volume1/steve/Courses/masterclass/
|
||||
Run on the Synology NAS.
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
import subprocess
|
||||
import sys
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
SOURCE = Path("/volume1/steve/Courses/masterclass")
|
||||
LIBRARY = Path("/volume1/steve/Eagle_Libraries/Masterclass.library")
|
||||
IMAGES = LIBRARY / "images"
|
||||
|
||||
SUPPORTED_EXTS = {
|
||||
"mp4", "mov", "m4v", "avi", "mkv", "webm", # video
|
||||
"mp3", "m4a", "aac", "wav", "flac", # audio
|
||||
"jpg", "jpeg", "png", "gif", "webp", # image
|
||||
"pdf", # docs
|
||||
}
|
||||
|
||||
VIDEO_EXTS = {"mp4", "mov", "m4v", "avi", "mkv", "webm"}
|
||||
IMAGE_EXTS = {"jpg", "jpeg", "png", "gif", "webp"}
|
||||
|
||||
# ID counter
|
||||
_id_counter = 0
|
||||
_id_prefix = f"MC{int(time.time()) % 10000000:07d}"
|
||||
|
||||
def gen_id():
|
||||
global _id_counter
|
||||
_id_counter += 1
|
||||
return f"{_id_prefix}{_id_counter:05d}"
|
||||
|
||||
def now_ms():
|
||||
return int(time.time() * 1000)
|
||||
|
||||
def get_video_info(filepath):
|
||||
"""Returns (width, height, duration) via ffprobe."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["ffprobe", "-v", "quiet", "-print_format", "json", "-show_streams", str(filepath)],
|
||||
capture_output=True, text=True, timeout=30
|
||||
)
|
||||
data = json.loads(result.stdout)
|
||||
width, height, duration = 0, 0, None
|
||||
for stream in data.get("streams", []):
|
||||
if stream.get("codec_type") in ("video", "audio"):
|
||||
w = stream.get("width", 0)
|
||||
h = stream.get("height", 0)
|
||||
if w and h:
|
||||
width, height = w, h
|
||||
dur = stream.get("duration")
|
||||
if dur:
|
||||
duration = float(dur)
|
||||
if width and duration:
|
||||
break
|
||||
return width, height, duration
|
||||
except Exception:
|
||||
return 0, 0, None
|
||||
|
||||
def generate_thumbnail(filepath, thumb_path, duration=None):
|
||||
"""Generate thumbnail PNG from video or image."""
|
||||
ext = filepath.suffix.lower().lstrip(".")
|
||||
try:
|
||||
if ext in VIDEO_EXTS:
|
||||
seek = "00:00:10"
|
||||
if duration and duration > 0:
|
||||
t = max(0, duration * 0.1)
|
||||
seek = f"{int(t // 3600):02d}:{int((t % 3600) // 60):02d}:{t % 60:05.2f}"
|
||||
subprocess.run(
|
||||
["ffmpeg", "-v", "quiet", "-ss", seek, "-i", str(filepath),
|
||||
"-vframes", "1", "-vf", "scale=320:-1", str(thumb_path)],
|
||||
timeout=60, capture_output=True
|
||||
)
|
||||
elif ext in IMAGE_EXTS:
|
||||
subprocess.run(
|
||||
["ffmpeg", "-v", "quiet", "-i", str(filepath),
|
||||
"-vf", "scale=320:-1", str(thumb_path)],
|
||||
timeout=30, capture_output=True
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def clean_name(filename_stem):
|
||||
"""Turn file stem into a readable display name."""
|
||||
name = filename_stem.replace("-", " ").replace("_", " ")
|
||||
# Strip leading numbering like "01 " or "1a "
|
||||
import re
|
||||
name = re.sub(r"^\d+[a-z]?\s+", "", name)
|
||||
return name.strip() or filename_stem
|
||||
|
||||
def collect_files():
|
||||
"""Walk source dir, return list of (filepath, category, course) tuples."""
|
||||
files = []
|
||||
for root, dirs, filenames in os.walk(SOURCE):
|
||||
dirs[:] = [d for d in sorted(dirs) if d != "@eaDir"]
|
||||
root_path = Path(root)
|
||||
# Determine category and course from path depth
|
||||
rel = root_path.relative_to(SOURCE)
|
||||
parts = rel.parts
|
||||
category = parts[0] if len(parts) >= 1 else None
|
||||
course = parts[1] if len(parts) >= 2 else None
|
||||
if category is None:
|
||||
continue
|
||||
for fname in sorted(filenames):
|
||||
ext = fname.rsplit(".", 1)[-1].lower() if "." in fname else ""
|
||||
if ext not in SUPPORTED_EXTS:
|
||||
continue
|
||||
files.append((root_path / fname, category, course))
|
||||
return files
|
||||
|
||||
def build_folder_tree(files):
|
||||
"""Build Eagle folder hierarchy from category/course structure."""
|
||||
# Collect unique category -> set of courses
|
||||
from collections import defaultdict, OrderedDict
|
||||
tree = defaultdict(set)
|
||||
for _, category, course in files:
|
||||
if category:
|
||||
tree[category].add(course or "")
|
||||
|
||||
folder_ids = {} # (category, course) -> id
|
||||
folders_json = []
|
||||
|
||||
for category in sorted(tree):
|
||||
cat_id = gen_id()
|
||||
folder_ids[(category, None)] = cat_id
|
||||
children = []
|
||||
for course in sorted(tree[category]):
|
||||
if not course:
|
||||
continue
|
||||
course_id = gen_id()
|
||||
folder_ids[(category, course)] = course_id
|
||||
children.append({
|
||||
"id": course_id,
|
||||
"name": course.replace("-", " ").replace("_", " ").title(),
|
||||
"description": "",
|
||||
"children": [],
|
||||
"modificationTime": now_ms(),
|
||||
"tags": [],
|
||||
"password": "",
|
||||
"passwordTips": "",
|
||||
})
|
||||
folders_json.append({
|
||||
"id": cat_id,
|
||||
"name": category.replace("-", " ").replace("_", " ").title(),
|
||||
"description": "",
|
||||
"children": children,
|
||||
"modificationTime": now_ms(),
|
||||
"tags": [],
|
||||
"password": "",
|
||||
"passwordTips": "",
|
||||
})
|
||||
|
||||
return folders_json, folder_ids
|
||||
|
||||
def main():
|
||||
print(f"Source: {SOURCE}")
|
||||
print(f"Library: {LIBRARY}")
|
||||
|
||||
if not SOURCE.exists():
|
||||
print("ERROR: Source directory not found.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# Collect files
|
||||
print("Scanning files...")
|
||||
files = collect_files()
|
||||
print(f"Found {len(files)} files to import.")
|
||||
|
||||
# Build folder tree
|
||||
folders_json, folder_ids = build_folder_tree(files)
|
||||
|
||||
# Create library structure
|
||||
IMAGES.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Write library metadata.json
|
||||
meta = {
|
||||
"folders": folders_json,
|
||||
"smartFolders": [],
|
||||
"quickAccess": [],
|
||||
"tagsGroups": [],
|
||||
"modificationTime": now_ms(),
|
||||
"applicationVersion": "4.0.0",
|
||||
}
|
||||
(LIBRARY / "metadata.json").write_text(json.dumps(meta))
|
||||
print("Library metadata written.")
|
||||
|
||||
# Import each file
|
||||
errors = []
|
||||
for i, (filepath, category, course) in enumerate(files, 1):
|
||||
try:
|
||||
ext = filepath.suffix.lower().lstrip(".")
|
||||
stem = filepath.stem
|
||||
display_name = clean_name(stem)
|
||||
item_id = gen_id()
|
||||
item_dir = IMAGES / f"{item_id}.info"
|
||||
item_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Hardlink file into .info dir to save space
|
||||
dest_file = item_dir / filepath.name
|
||||
try:
|
||||
os.link(filepath, dest_file)
|
||||
except OSError:
|
||||
import shutil
|
||||
shutil.copy2(filepath, dest_file)
|
||||
|
||||
# Get metadata
|
||||
stat = filepath.stat()
|
||||
size = stat.st_size
|
||||
btime_ms = int(stat.st_ctime * 1000)
|
||||
mtime_ms = int(stat.st_mtime * 1000)
|
||||
|
||||
width, height, duration = 0, 0, None
|
||||
if ext in VIDEO_EXTS:
|
||||
width, height, duration = get_video_info(filepath)
|
||||
elif ext in IMAGE_EXTS:
|
||||
width, height, _ = get_video_info(filepath)
|
||||
|
||||
# Generate thumbnail
|
||||
thumb_name = f"{stem}_thumbnail.png"
|
||||
thumb_path = item_dir / thumb_name
|
||||
generate_thumbnail(filepath, thumb_path, duration)
|
||||
has_thumb = thumb_path.exists()
|
||||
|
||||
# Determine folder assignment
|
||||
folder_id = None
|
||||
if category and course:
|
||||
folder_id = folder_ids.get((category, course))
|
||||
if not folder_id and category:
|
||||
folder_id = folder_ids.get((category, None))
|
||||
|
||||
# Write metadata.json
|
||||
img_meta = {
|
||||
"id": item_id,
|
||||
"name": display_name,
|
||||
"size": size,
|
||||
"btime": btime_ms,
|
||||
"mtime": mtime_ms,
|
||||
"ext": ext,
|
||||
"tags": [],
|
||||
"folders": [folder_id] if folder_id else [],
|
||||
"isDeleted": False,
|
||||
"url": "",
|
||||
"annotation": "",
|
||||
"modificationTime": now_ms(),
|
||||
"width": width,
|
||||
"height": height,
|
||||
"resolutionWidth": width,
|
||||
"resolutionHeight": height,
|
||||
"lastModified": now_ms(),
|
||||
}
|
||||
if duration is not None:
|
||||
img_meta["duration"] = duration
|
||||
|
||||
(item_dir / "metadata.json").write_text(json.dumps(img_meta))
|
||||
|
||||
if i % 50 == 0 or i == len(files):
|
||||
print(f"Progress: {i}/{len(files)}", flush=True)
|
||||
|
||||
except Exception as e:
|
||||
errors.append(f"{filepath}: {e}")
|
||||
print(f"ERROR: {filepath}: {e}", file=sys.stderr)
|
||||
|
||||
print(f"\n=== Import complete ===")
|
||||
print(f"Items imported: {i}")
|
||||
print(f"Errors: {len(errors)}")
|
||||
if errors:
|
||||
for e in errors[:10]:
|
||||
print(f" {e}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
214
import_masterclass.sh
Normal file
214
import_masterclass.sh
Normal file
@ -0,0 +1,214 @@
|
||||
#!/bin/bash
|
||||
# import_masterclass.sh
|
||||
# Creates a Masterclass.library in Eagle format from /volume1/steve/Courses/masterclass/
|
||||
# Run this ON the Synology NAS (ssh in first)
|
||||
#
|
||||
# Eagle library structure:
|
||||
# Masterclass.library/
|
||||
# metadata.json <- library-level metadata (folders tree)
|
||||
# images/
|
||||
# <ID>.info/
|
||||
# <original-file>
|
||||
# <name>_thumbnail.png (generated by ffmpeg for video; copied for images)
|
||||
# metadata.json
|
||||
|
||||
set -e
|
||||
|
||||
SOURCE="/volume1/steve/Courses/masterclass"
|
||||
LIBRARY="/volume1/steve/Eagle_Libraries/Masterclass.library"
|
||||
IMAGES="$LIBRARY/images"
|
||||
|
||||
# Timestamp-based unique ID generator (Eagle uses alphanumeric IDs)
|
||||
# We'll use a counter-based approach with a base36-like prefix
|
||||
ID_COUNTER=0
|
||||
BASE_ID="MC$(date +%s | tail -c 8)"
|
||||
|
||||
gen_id() {
|
||||
ID_COUNTER=$((ID_COUNTER + 1))
|
||||
printf '%s%06d' "$BASE_ID" "$ID_COUNTER"
|
||||
}
|
||||
|
||||
# Get video duration in seconds via ffprobe
|
||||
get_duration() {
|
||||
local file="$1"
|
||||
ffprobe -v quiet -print_format json -show_streams "$file" 2>/dev/null \
|
||||
| grep '"duration"' | head -1 | grep -oE '[0-9]+(\.[0-9]+)?' | head -1 || echo "0"
|
||||
}
|
||||
|
||||
# Get image/video dimensions
|
||||
get_dimensions() {
|
||||
local file="$1"
|
||||
ffprobe -v quiet -print_format json -show_streams "$file" 2>/dev/null \
|
||||
| grep -E '"width"|"height"' | head -2 \
|
||||
| grep -oE '[0-9]+' | head -2 | tr '\n' 'x' | sed 's/x$//' || echo "0x0"
|
||||
}
|
||||
|
||||
# Mime/ext helpers
|
||||
ext_of() { echo "${1##*.}" | tr '[:upper:]' '[:lower:]'; }
|
||||
|
||||
is_video() {
|
||||
case "$(ext_of "$1")" in mp4|mov|m4v|avi|mkv|webm) return 0;; esac; return 1
|
||||
}
|
||||
is_image() {
|
||||
case "$(ext_of "$1")" in jpg|jpeg|png|gif|webp|svg) return 0;; esac; return 1
|
||||
}
|
||||
is_audio() {
|
||||
case "$(ext_of "$1")" in mp3|m4a|aac|wav|flac) return 0;; esac; return 1
|
||||
}
|
||||
|
||||
echo "=== Creating Masterclass Eagle Library ==="
|
||||
echo "Source: $SOURCE"
|
||||
echo "Library: $LIBRARY"
|
||||
|
||||
# --- Build folder tree from directory structure ---
|
||||
# Walk category/course dirs and assign IDs
|
||||
declare -A FOLDER_IDS # path -> id
|
||||
|
||||
FOLDERS_JSON=""
|
||||
|
||||
build_folder_json() {
|
||||
local dirpath="$1"
|
||||
local name="$2"
|
||||
local children_json="$3"
|
||||
local fid="${FOLDER_IDS[$dirpath]}"
|
||||
local now=$(date +%s%3N)
|
||||
echo "{\"id\":\"$fid\",\"name\":\"$name\",\"description\":\"\",\"children\":[$children_json],\"modificationTime\":$now,\"tags\":[],\"password\":\"\",\"passwordTips\":\"\"}"
|
||||
}
|
||||
|
||||
# Pre-assign IDs to all category and course folders
|
||||
for category_dir in "$SOURCE"/*/; do
|
||||
[[ "$(basename "$category_dir")" == "@eaDir" ]] && continue
|
||||
category="$(basename "$category_dir")"
|
||||
FOLDER_IDS["$category_dir"]="$(gen_id)"
|
||||
for course_dir in "$category_dir"*/; do
|
||||
[[ "$(basename "$course_dir")" == "@eaDir" ]] && continue
|
||||
[[ ! -d "$course_dir" ]] && continue
|
||||
# Some courses have sub-subdirs (like mp3/, guide/) — treat those as leaf
|
||||
FOLDER_IDS["$course_dir"]="$(gen_id)"
|
||||
done
|
||||
done
|
||||
|
||||
# Build nested folder JSON
|
||||
TOP_FOLDERS=""
|
||||
for category_dir in "$SOURCE"/*/; do
|
||||
[[ "$(basename "$category_dir")" == "@eaDir" ]] && continue
|
||||
category="$(basename "$category_dir")"
|
||||
cat_id="${FOLDER_IDS[$category_dir]}"
|
||||
COURSE_FOLDERS=""
|
||||
for course_dir in "$category_dir"*/; do
|
||||
[[ "$(basename "$course_dir")" == "@eaDir" ]] && continue
|
||||
[[ ! -d "$course_dir" ]] && continue
|
||||
course="$(basename "$course_dir")"
|
||||
course_id="${FOLDER_IDS[$course_dir]}"
|
||||
now=$(date +%s%3N)
|
||||
course_json="{\"id\":\"$course_id\",\"name\":\"$course\",\"description\":\"\",\"children\":[],\"modificationTime\":$now,\"tags\":[],\"password\":\"\",\"passwordTips\":\"\"}"
|
||||
COURSE_FOLDERS="${COURSE_FOLDERS:+$COURSE_FOLDERS,}$course_json"
|
||||
done
|
||||
now=$(date +%s%3N)
|
||||
cat_json="{\"id\":\"$cat_id\",\"name\":\"$category\",\"description\":\"\",\"children\":[$COURSE_FOLDERS],\"modificationTime\":$now,\"tags\":[],\"password\":\"\",\"passwordTips\":\"\"}"
|
||||
TOP_FOLDERS="${TOP_FOLDERS:+$TOP_FOLDERS,}$cat_json"
|
||||
done
|
||||
|
||||
# --- Create library structure ---
|
||||
mkdir -p "$IMAGES"
|
||||
|
||||
now=$(date +%s%3N)
|
||||
cat > "$LIBRARY/metadata.json" <<EOF
|
||||
{"folders":[$TOP_FOLDERS],"smartFolders":[],"quickAccess":[],"tagsGroups":[],"modificationTime":$now,"applicationVersion":"4.0.0"}
|
||||
EOF
|
||||
|
||||
echo "Library metadata written."
|
||||
|
||||
# --- Import files ---
|
||||
TOTAL=0
|
||||
DONE=0
|
||||
SKIPPED=0
|
||||
|
||||
# Count total files to import
|
||||
TOTAL=$(find "$SOURCE" -type f -not -path '*/@eaDir/*' \
|
||||
\( -name "*.mp4" -o -name "*.mov" -o -name "*.m4v" \
|
||||
-o -name "*.mp3" -o -name "*.m4a" \
|
||||
-o -name "*.jpg" -o -name "*.jpeg" -o -name "*.png" -o -name "*.webp" \
|
||||
-o -name "*.pdf" \) | wc -l)
|
||||
echo "Total files to import: $TOTAL"
|
||||
|
||||
find "$SOURCE" -type f -not -path '*/@eaDir/*' \
|
||||
\( -name "*.mp4" -o -name "*.mov" -o -name "*.m4v" \
|
||||
-o -name "*.mp3" -o -name "*.m4a" \
|
||||
-o -name "*.jpg" -o -name "*.jpeg" -o -name "*.png" -o -name "*.webp" \
|
||||
-o -name "*.pdf" \) | sort | while read filepath; do
|
||||
|
||||
filename="$(basename "$filepath")"
|
||||
ext="$(ext_of "$filepath")"
|
||||
# Name without extension
|
||||
name="${filename%.*}"
|
||||
# Clean up name: replace underscores/hyphens with spaces, strip leading numbers
|
||||
pretty_name="$(echo "$name" | sed 's/[-_]/ /g' | sed 's/^ *[0-9]*[a-z]* //')"
|
||||
[[ -z "$pretty_name" ]] && pretty_name="$name"
|
||||
|
||||
# Determine folder ID from path
|
||||
# Path: $SOURCE/<category>/<course>/... or $SOURCE/<category>/<course>/<subdir>/...
|
||||
rel="${filepath#$SOURCE/}"
|
||||
category="$(echo "$rel" | cut -d'/' -f1)"
|
||||
course="$(echo "$rel" | cut -d'/' -f2)"
|
||||
course_dir="$SOURCE/$category/$course/"
|
||||
folder_id="${FOLDER_IDS[$course_dir]}"
|
||||
[[ -z "$folder_id" ]] && folder_id="${FOLDER_IDS[$SOURCE/$category/]}"
|
||||
|
||||
# Generate item ID
|
||||
item_id="$(gen_id)"
|
||||
item_dir="$IMAGES/${item_id}.info"
|
||||
mkdir -p "$item_dir"
|
||||
|
||||
# Hardlink the file into the .info dir (saves space)
|
||||
ln "$filepath" "$item_dir/$filename" 2>/dev/null || cp "$filepath" "$item_dir/$filename"
|
||||
|
||||
# Get metadata
|
||||
size=$(stat -c%s "$filepath" 2>/dev/null || echo 0)
|
||||
btime=$(stat -c%W "$filepath" 2>/dev/null || date +%s)
|
||||
mtime=$(stat -c%Y "$filepath" 2>/dev/null || date +%s)
|
||||
btime_ms=$((btime * 1000))
|
||||
mtime_ms=$((mtime * 1000))
|
||||
now_ms=$(date +%s%3N)
|
||||
|
||||
width=0; height=0; duration_val="null"
|
||||
|
||||
if is_video "$filepath"; then
|
||||
dims=$(get_dimensions "$filepath")
|
||||
width=$(echo "$dims" | cut -dx -f1)
|
||||
height=$(echo "$dims" | cut -dx -f2)
|
||||
dur=$(get_duration "$filepath")
|
||||
[[ -n "$dur" && "$dur" != "0" ]] && duration_val="$dur"
|
||||
# Generate thumbnail at 10% into video
|
||||
thumb_file="$item_dir/${name}_thumbnail.png"
|
||||
ffmpeg -v quiet -ss "$(echo "$dur * 0.1" | awk '{printf "%.0f", $1}')" \
|
||||
-i "$filepath" -vframes 1 -vf "scale=320:-1" "$thumb_file" 2>/dev/null || true
|
||||
elif is_image "$filepath"; then
|
||||
dims=$(get_dimensions "$filepath")
|
||||
width=$(echo "$dims" | cut -dx -f1)
|
||||
height=$(echo "$dims" | cut -dx -f2)
|
||||
# Use the image itself as thumbnail (resized)
|
||||
thumb_file="$item_dir/${name}_thumbnail.png"
|
||||
ffmpeg -v quiet -i "$filepath" -vf "scale=320:-1" "$thumb_file" 2>/dev/null || \
|
||||
cp "$filepath" "$thumb_file" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# Build metadata.json
|
||||
folder_arr="[]"
|
||||
[[ -n "$folder_id" ]] && folder_arr="[\"$folder_id\"]"
|
||||
|
||||
cat > "$item_dir/metadata.json" <<METAEOF
|
||||
{"id":"$item_id","name":"$pretty_name","size":$size,"btime":$btime_ms,"mtime":$mtime_ms,"ext":"$ext","tags":[],"folders":$folder_arr,"isDeleted":false,"url":"","annotation":"","modificationTime":$now_ms,"width":$width,"height":$height,"resolutionWidth":$width,"resolutionHeight":$height${duration_val:+,"duration":$duration_val},"lastModified":$now_ms}
|
||||
METAEOF
|
||||
|
||||
DONE=$((DONE + 1))
|
||||
if (( DONE % 50 == 0 )); then
|
||||
echo "Progress: $DONE / $TOTAL"
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "=== Import complete ==="
|
||||
echo "Library: $LIBRARY"
|
||||
ls "$IMAGES" | wc -l
|
||||
echo "items imported."
|
||||
Loading…
x
Reference in New Issue
Block a user