Data Import - Comprehensive Import Pipeline¶

from src.data.import_pipeline import (
    ImportConfiguration, 
    DataImportPipeline,
    quick_import_by_search
)

# Quick search import
progress = quick_import_by_search(
    search_query="computer vision",
    max_papers=200,
    progress_callback=lambda p: print(f"Progress: {p.overall_progress_percent:.1f}%")
)

print(f"Import completed: {progress.processed_papers} papers imported")

from src.data.import_pipeline import ImportConfiguration, DataImportPipeline

# Create detailed configuration
config = ImportConfiguration(
    search_query="natural language processing",
    max_papers=1000,
    batch_size=75,
    include_citations=True,
    include_authors=True,
    include_venues=True,
    min_citation_count=10,
    year_range=(2018, 2024),
    api_delay=1.2,
    save_progress=True,
    max_workers=4
)

# Execute import with configuration
pipeline = DataImportPipeline(config)
progress = pipeline.import_papers_by_search(
    search_query=config.search_query,
    max_papers=config.max_papers
)

# Monitor results
print(f"Status: {progress.status}")
print(f"Papers: {progress.processed_papers}/{progress.total_papers}")
print(f"Citations: {progress.citations_created}")
print(f"Authors: {progress.authors_created}")

def detailed_progress_callback(progress):
    """Comprehensive progress monitoring"""
    print(f"Status: {progress.status.value}")
    print(f"Overall Progress: {progress.overall_progress_percent:.1f}%")
    print(f"Current Batch: {progress.current_batch_progress_percent:.1f}%")
    print(f"Performance: {progress.papers_per_second:.2f} papers/sec")
    print(f"Errors: {progress.error_count}")
    if progress.errors:
        print("Recent errors:")
        for error in progress.errors[-3:]:  # Show last 3 errors
            print(f"  - {error}")
    print("---")

# Use callback for detailed monitoring
progress = quick_import_by_search(
    search_query="reinforcement learning",
    max_papers=500,
    progress_callback=detailed_progress_callback
)

# Example error tracking
progress = pipeline.get_current_progress()

print(f"Total errors: {progress.error_count}")
print(f"Error rate: {progress.error_rate:.2%}")

# Review recent errors
for error in progress.errors[-5:]:
    print(f"Error: {error.message}")
    print(f"Type: {error.error_type}")
    print(f"Paper ID: {error.paper_id}")
    print(f"Timestamp: {error.timestamp}")
    print("---")

from src.data.import_pipeline import DataImportPipeline

# Streaming is automatically enabled for search imports
pipeline = DataImportPipeline(config)

# Monitor streaming progress
def streaming_callback(progress):
    print(f"Fetching: {progress.current_fetch_progress:.1f}%")
    print(f"Processing: {progress.current_batch_progress:.1f}%")

pipeline.add_progress_callback(streaming_callback)

config = ImportConfiguration(
    search_query="large scale import",
    max_papers=5000,
    save_progress=True,
    progress_file="large_import_progress.json"  # Optional custom path
)

# If import is interrupted, restart with same configuration
# Pipeline automatically resumes from last checkpoint
pipeline = DataImportPipeline(config)
progress = pipeline.resume_import()  # Resumes if checkpoint exists

config = ImportConfiguration(
    max_papers=2000,
    batch_size=100,  # Starting batch size
    adaptive_batching=True,  # Enable intelligent adjustments
    max_batch_size=200,      # Upper limit for batch adjustments
    min_batch_size=25        # Lower limit for error recovery
)

# Access quality metrics after import
quality_report = pipeline.get_quality_report()

print(f"Data completeness: {quality_report.completeness_score:.1%}")
print(f"Citation coverage: {quality_report.citation_coverage:.1%}")
print(f"Author match rate: {quality_report.author_match_rate:.1%}")
print(f"Venue match rate: {quality_report.venue_match_rate:.1%}")

from src.data.import_pipeline import quick_import_by_search
from src.services.ml_service import get_ml_service

# Import data
progress = quick_import_by_search("machine learning", max_papers=500)

# Immediately use for ML predictions
ml_service = get_ml_service()
if progress.status == ImportStatus.COMPLETED:
    # Train model with new data
    model = ml_service.train_model()

    # Generate predictions
    predictions = ml_service.predict_citations(
        paper_id="some_imported_paper_id",
        top_k=10
    )

from src.data.import_pipeline import quick_import_by_search
from src.services.analytics_service import get_analytics_service

# Import domain-specific papers
progress = quick_import_by_search("computer vision", max_papers=1000)

# Analyze imported network
analytics = get_analytics_service()
if progress.status == ImportStatus.COMPLETED:
    # Community detection
    communities = analytics.detect_communities()

    # Network metrics
    metrics = analytics.compute_network_metrics()

    # Export analysis
    report = analytics.generate_network_report()

from src.data.import_pipeline import DataImportPipeline, ImportConfiguration

queries = [
    "deep learning computer vision",
    "machine learning natural language processing", 
    "reinforcement learning robotics"
]

all_results = []
for query in queries:
    config = ImportConfiguration(
        search_query=query,
        max_papers=300,
        min_citation_count=15
    )

    pipeline = DataImportPipeline(config)
    progress = pipeline.import_papers_by_search(query, 300)
    all_results.append(progress)

# Combine results for unified analysis
total_papers = sum(p.processed_papers for p in all_results)
print(f"Multi-query import completed: {total_papers} papers total")

from src.data.import_pipeline import DataImportPipeline
from datetime import datetime, timedelta

# Import recent papers only
recent_date = datetime.now() - timedelta(days=30)
config = ImportConfiguration(
    search_query="latest research",
    min_year=recent_date.year,
    max_papers=200,
    include_citations=True
)

# Regular update workflow
pipeline = DataImportPipeline(config)
progress = pipeline.import_papers_by_search("latest research", 200)

# Update existing ML models with new data
if progress.processed_papers > 0:
    ml_service.update_model_with_new_data()

import logging
from src.data.import_pipeline import DataImportPipeline

# Enable debug logging
logging.basicConfig(level=logging.DEBUG)

# Create pipeline with debug options
config = ImportConfiguration(
    search_query="debug import",
    max_papers=5,
    debug_mode=True,
    verbose_errors=True
)

pipeline = DataImportPipeline(config)

# Monitor detailed progress
def debug_callback(progress):
    print(f"DEBUG: {progress.debug_info}")
    if progress.current_error:
        print(f"Current error: {progress.current_error}")

progress = pipeline.import_papers_by_search("debug", 5, debug_callback)