| """ |
| Advanced 3D Reconstruction from Single Images with Responsible AI Features |
| |
| """ |
|
|
| import gradio as gr |
| import numpy as np |
| import torch |
| from PIL import Image |
| from transformers import GLPNForDepthEstimation, GLPNImageProcessor |
| import open3d as o3d |
| import plotly.graph_objects as go |
| import matplotlib.pyplot as plt |
| import io |
| import json |
| import time |
| from pathlib import Path |
| import tempfile |
| import zipfile |
| import hashlib |
| from datetime import datetime |
|
|
| |
| |
| |
|
|
| RESPONSIBLE_AI_NOTICE = """ |
| ## ⚠️ Responsible Use Guidelines |
| |
| ### Privacy & Consent |
| - **Do not upload images containing identifiable people without their explicit consent** |
| - **Do not use for surveillance, tracking, or monitoring individuals** |
| - Facial features may be reconstructed in 3D - consider privacy implications |
| - Remove metadata (EXIF) that may contain location or personal information |
| |
| ### Ethical Use |
| - This tool is for **educational, research, and creative purposes only** |
| - **Prohibited uses:** |
| - Creating deepfakes or misleading 3D content |
| - Unauthorized documentation of private property |
| - Circumventing security systems |
| - Generating 3D models for harassment or stalking |
| - Commercial use without proper rights to source images |
| |
| ### Limitations & Bias |
| - Models trained primarily on indoor Western architecture |
| - May perform poorly on non-Western architectural styles |
| - Scale is relative, not absolute - not suitable for precision measurements |
| - Single viewpoint limitations - occluded areas are inferred, not captured |
| |
| ### Data Usage |
| - Images are processed locally during your session |
| - No images are stored or transmitted to external servers |
| - Processing logs contain only technical metrics, no image content |
| - You retain all rights to your uploaded images and generated 3D models |
| |
| |
| **By using this tool, you agree to these responsible use guidelines.** |
| """ |
|
|
| |
| |
| |
|
|
| def check_image_safety(image): |
| """Basic safety checks for uploaded images""" |
| warnings = [] |
| |
| width, height = image.size |
| if width * height > 10_000_000: |
| warnings.append("⚠️ Very large image - consider resizing to improve processing speed") |
| |
| aspect_ratio = max(width, height) / min(width, height) |
| if aspect_ratio > 3: |
| warnings.append("⚠️ Unusual aspect ratio detected - ensure image doesn't contain unintended content") |
| |
| try: |
| exif = image.getexif() |
| if exif: |
| has_gps = any(k for k in exif.keys() if k in [34853, 0x8825]) |
| if has_gps: |
| warnings.append("⚠️ GPS location data detected in image - consider removing EXIF data for privacy") |
| except: |
| pass |
| |
| return True, "\n".join(warnings) if warnings else None |
|
|
| def generate_session_id(): |
| """Generate anonymous session ID for logging""" |
| return hashlib.sha256(str(datetime.now()).encode()).hexdigest()[:16] |
|
|
| def content_policy_check(image): |
| """Check if image content violates usage policies""" |
| width, height = image.size |
| |
| if width < 100 or height < 100: |
| return False, "Image too small - minimum 100x100 pixels required for meaningful reconstruction" |
| |
| return True, None |
|
|
| |
| |
| |
|
|
| print("Loading GLPN model (lightweight)...") |
| try: |
| glpn_processor = GLPNImageProcessor.from_pretrained("vinvino02/glpn-nyu") |
| glpn_model = GLPNForDepthEstimation.from_pretrained("vinvino02/glpn-nyu") |
| print("✓ GLPN model loaded successfully!") |
| except Exception as e: |
| print(f"Error loading model: {e}") |
| glpn_processor = None |
| glpn_model = None |
|
|
| |
| dpt_model = None |
| dpt_processor = None |
|
|
| |
| |
| |
|
|
| def process_image(image, model_choice="GLPN (Recommended)", visualization_type="mesh"): |
| """Optimized processing pipeline""" |
| |
| def _generate_quality_assessment(metrics): |
| assessment = [] |
| outlier_pct = (metrics['outliers_removed'] / metrics['initial_points']) * 100 |
| |
| if outlier_pct < 5: |
| assessment.append("Very clean depth estimation") |
| elif outlier_pct < 15: |
| assessment.append("Good depth quality") |
| else: |
| assessment.append("High noise in depth estimation") |
| |
| if metrics['is_edge_manifold'] and metrics['is_vertex_manifold']: |
| assessment.append("Excellent topology") |
| elif metrics['is_vertex_manifold']: |
| assessment.append("Good local topology") |
| else: |
| assessment.append("Topology issues present") |
| |
| if metrics['is_watertight']: |
| assessment.append("Watertight mesh - ready for 3D printing!") |
| else: |
| assessment.append("Not watertight - needs repair for 3D printing") |
| |
| return "\n".join(f"- {item}" for item in assessment) |
| |
| if glpn_model is None: |
| return None, None, None, "❌ Model failed to load. Please refresh the page.", None |
| |
| try: |
| print("Starting reconstruction...") |
| |
| |
| new_height = 480 if image.height > 480 else image.height |
| new_height -= (new_height % 32) |
| new_width = int(new_height * image.width / image.height) |
| diff = new_width % 32 |
| new_width = new_width - diff if diff < 16 else new_width + (32 - diff) |
| new_size = (new_width, new_height) |
| image = image.resize(new_size, Image.LANCZOS) |
| |
| |
| if model_choice == "GLPN (Recommended)": |
| processor = glpn_processor |
| model = glpn_model |
| else: |
| global dpt_model, dpt_processor |
| if dpt_model is None: |
| print("Loading DPT model (first time only)...") |
| from transformers import DPTForDepthEstimation, DPTImageProcessor |
| dpt_processor = DPTImageProcessor.from_pretrained("Intel/dpt-large") |
| dpt_model = DPTForDepthEstimation.from_pretrained("Intel/dpt-large") |
| print("✓ DPT model loaded!") |
| processor = dpt_processor |
| model = dpt_model |
| |
| inputs = processor(images=image, return_tensors="pt") |
| |
| start_time = time.time() |
| with torch.no_grad(): |
| outputs = model(**inputs) |
| predicted_depth = outputs.predicted_depth |
| depth_time = time.time() - start_time |
| |
| |
| pad = 16 |
| output = predicted_depth.squeeze().cpu().numpy() * 1000.0 |
| output = output[pad:-pad, pad:-pad] |
| image_cropped = image.crop((pad, pad, image.width - pad, image.height - pad)) |
| |
| depth_height, depth_width = output.shape |
| img_width, img_height = image_cropped.size |
| |
| if depth_height != img_height or depth_width != img_width: |
| from scipy import ndimage |
| zoom_factors = (img_height / depth_height, img_width / depth_width) |
| output = ndimage.zoom(output, zoom_factors, order=1) |
| |
| image = image_cropped |
| |
| |
| fig, ax = plt.subplots(1, 2, figsize=(14, 7)) |
| ax[0].imshow(image) |
| ax[0].set_title('Original Image', fontsize=14, fontweight='bold') |
| ax[0].axis('off') |
| |
| im = ax[1].imshow(output, cmap='plasma') |
| ax[1].set_title('Estimated Depth Map', fontsize=14, fontweight='bold') |
| ax[1].axis('off') |
| plt.colorbar(im, ax=ax[1], fraction=0.046, pad=0.04) |
| plt.tight_layout() |
| |
| buf = io.BytesIO() |
| plt.savefig(buf, format='png', dpi=150, bbox_inches='tight') |
| buf.seek(0) |
| depth_viz = Image.open(buf) |
| plt.close() |
| |
| |
| width, height = image.size |
| |
| if output.shape != (height, width): |
| from scipy import ndimage |
| zoom_factors = (height / output.shape[0], width / output.shape[1]) |
| output = ndimage.zoom(output, zoom_factors, order=1) |
| |
| depth_image = (output * 255 / np.max(output)).astype(np.uint8) |
| image_array = np.array(image) |
| |
| depth_o3d = o3d.geometry.Image(depth_image) |
| image_o3d = o3d.geometry.Image(image_array) |
| rgbd_image = o3d.geometry.RGBDImage.create_from_color_and_depth( |
| image_o3d, depth_o3d, convert_rgb_to_intensity=False |
| ) |
| |
| camera_intrinsic = o3d.camera.PinholeCameraIntrinsic() |
| camera_intrinsic.set_intrinsics(width, height, 500, 500, width/2, height/2) |
| |
| pcd = o3d.geometry.PointCloud.create_from_rgbd_image(rgbd_image, camera_intrinsic) |
| initial_points = len(pcd.points) |
| |
| |
| cl, ind = pcd.remove_statistical_outlier(nb_neighbors=20, std_ratio=2.0) |
| pcd = pcd.select_by_index(ind) |
| outliers_removed = initial_points - len(pcd.points) |
| |
| |
| pcd.estimate_normals() |
| pcd.orient_normals_to_align_with_direction() |
| |
| |
| mesh_start = time.time() |
| mesh = o3d.geometry.TriangleMesh.create_from_point_cloud_poisson( |
| pcd, depth=9, n_threads=1 |
| )[0] |
| |
| |
| pcd_tree = o3d.geometry.KDTreeFlann(pcd) |
| mesh_colors = [] |
| for vertex in mesh.vertices: |
| [_, idx, _] = pcd_tree.search_knn_vector_3d(vertex, 1) |
| mesh_colors.append(pcd.colors[idx[0]]) |
| mesh.vertex_colors = o3d.utility.Vector3dVector(np.array(mesh_colors)) |
| |
| rotation = mesh.get_rotation_matrix_from_xyz((np.pi, 0, 0)) |
| mesh.rotate(rotation, center=(0, 0, 0)) |
| mesh_time = time.time() - mesh_start |
| |
| |
| mesh.compute_vertex_normals() |
| |
| metrics = { |
| 'model_used': model_choice, |
| 'depth_estimation_time': f"{depth_time:.2f}s", |
| 'mesh_reconstruction_time': f"{mesh_time:.2f}s", |
| 'total_time': f"{depth_time + mesh_time:.2f}s", |
| 'initial_points': initial_points, |
| 'outliers_removed': outliers_removed, |
| 'final_points': len(pcd.points), |
| 'vertices': len(mesh.vertices), |
| 'triangles': len(mesh.triangles), |
| 'is_edge_manifold': mesh.is_edge_manifold(), |
| 'is_vertex_manifold': mesh.is_vertex_manifold(), |
| 'is_watertight': mesh.is_watertight(), |
| } |
| |
| |
| try: |
| surface_area = mesh.get_surface_area() |
| if surface_area > 0: |
| metrics['surface_area'] = float(surface_area) |
| else: |
| vertices = np.asarray(mesh.vertices) |
| triangles = np.asarray(mesh.triangles) |
| v0 = vertices[triangles[:, 0]] |
| v1 = vertices[triangles[:, 1]] |
| v2 = vertices[triangles[:, 2]] |
| cross = np.cross(v1 - v0, v2 - v0) |
| areas = 0.5 * np.linalg.norm(cross, axis=1) |
| metrics['surface_area'] = float(np.sum(areas)) |
| except: |
| metrics['surface_area'] = "Unable to compute" |
| |
| |
| try: |
| if mesh.is_watertight(): |
| metrics['volume'] = float(mesh.get_volume()) |
| else: |
| metrics['volume'] = None |
| except: |
| metrics['volume'] = None |
| |
| |
| points = np.asarray(pcd.points) |
| colors = np.asarray(pcd.colors) |
| |
| if visualization_type == "point_cloud": |
| scatter = go.Scatter3d( |
| x=points[:, 0], y=points[:, 1], z=points[:, 2], |
| mode='markers', |
| marker=dict( |
| size=2, |
| color=['rgb({},{},{})'.format(int(r*255), int(g*255), int(b*255)) |
| for r, g, b in colors], |
| ), |
| name='Point Cloud' |
| ) |
| |
| plotly_fig = go.Figure(data=[scatter]) |
| plotly_fig.update_layout( |
| scene=dict( |
| xaxis=dict(visible=False), |
| yaxis=dict(visible=False), |
| zaxis=dict(visible=False), |
| aspectmode='data' |
| ), |
| height=700, |
| title="Point Cloud" |
| ) |
| else: |
| vertices = np.asarray(mesh.vertices) |
| triangles = np.asarray(mesh.triangles) |
| |
| if mesh.has_vertex_colors(): |
| vertex_colors = np.asarray(mesh.vertex_colors) |
| colors_rgb = ['rgb({},{},{})'.format(int(r*255), int(g*255), int(b*255)) |
| for r, g, b in vertex_colors] |
| |
| mesh_trace = go.Mesh3d( |
| x=vertices[:, 0], y=vertices[:, 1], z=vertices[:, 2], |
| i=triangles[:, 0], j=triangles[:, 1], k=triangles[:, 2], |
| vertexcolor=colors_rgb, |
| opacity=0.95 |
| ) |
| else: |
| mesh_trace = go.Mesh3d( |
| x=vertices[:, 0], y=vertices[:, 1], z=vertices[:, 2], |
| i=triangles[:, 0], j=triangles[:, 1], k=triangles[:, 2], |
| color='lightblue', |
| opacity=0.9 |
| ) |
| |
| plotly_fig = go.Figure(data=[mesh_trace]) |
| plotly_fig.update_layout( |
| scene=dict( |
| xaxis=dict(visible=False), |
| yaxis=dict(visible=False), |
| zaxis=dict(visible=False), |
| aspectmode='data' |
| ), |
| height=700, |
| title="3D Mesh" |
| ) |
| |
| |
| temp_dir = tempfile.mkdtemp() |
| |
| pcd_path = Path(temp_dir) / "point_cloud.ply" |
| o3d.io.write_point_cloud(str(pcd_path), pcd) |
| |
| mesh_path = Path(temp_dir) / "mesh.ply" |
| o3d.io.write_triangle_mesh(str(mesh_path), mesh) |
| |
| mesh_obj_path = Path(temp_dir) / "mesh.obj" |
| o3d.io.write_triangle_mesh(str(mesh_obj_path), mesh) |
| |
| mesh_stl_path = Path(temp_dir) / "mesh.stl" |
| o3d.io.write_triangle_mesh(str(mesh_stl_path), mesh) |
| |
| metrics_path = Path(temp_dir) / "metrics.json" |
| with open(metrics_path, 'w') as f: |
| json.dump(metrics, f, indent=2, default=str) |
| |
| zip_path = Path(temp_dir) / "reconstruction_complete.zip" |
| with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf: |
| zipf.write(pcd_path, pcd_path.name) |
| zipf.write(mesh_path, mesh_path.name) |
| zipf.write(mesh_obj_path, mesh_obj_path.name) |
| zipf.write(mesh_stl_path, mesh_stl_path.name) |
| zipf.write(metrics_path, metrics_path.name) |
| |
| assessment = _generate_quality_assessment(metrics) |
| |
| report = f""" |
| ## Reconstruction Complete! |
| |
| ### Performance |
| - **Processing Time**: {metrics['total_time']} |
| - **Points**: {metrics['final_points']:,} |
| - **Triangles**: {metrics['triangles']:,} |
| |
| ### Quality |
| - **Topology**: {'Good' if metrics['is_vertex_manifold'] else 'Issues'} |
| - **Watertight**: {'Yes' if metrics['is_watertight'] else 'No'} |
| |
| ### Assessment |
| {assessment} |
| |
| **Download the complete package below!** |
| """ |
| |
| return depth_viz, plotly_fig, str(zip_path), report, json.dumps(metrics, indent=2, default=str) |
| |
| except Exception as e: |
| import traceback |
| return None, None, None, f"Error: {str(e)}\n\n{traceback.format_exc()}", None |
|
|
| def process_image_with_safeguards(image, model_choice="GLPN (Recommended)", visualization_type="mesh", consent_given=False): |
| """Main processing with safeguards""" |
| session_id = generate_session_id() |
| |
| if not consent_given: |
| return None, None, None, "**You must agree to the Responsible Use Guidelines first.**", None |
| |
| if image is None: |
| return None, None, None, "Please upload an image first.", None |
| |
| is_safe, safety_warning = check_image_safety(image) |
| passes_policy, policy_message = content_policy_check(image) |
| |
| if not passes_policy: |
| return None, None, None, f"{policy_message}", None |
| |
| try: |
| result = process_image(image, model_choice, visualization_type) |
| depth_viz, plotly_fig, zip_path, report, json_metrics = result |
| |
| if safety_warning: |
| report = f"**Privacy Notice:**\n{safety_warning}\n\n{report}" |
| |
| metrics = json.loads(json_metrics) |
| metrics['responsible_ai'] = { |
| 'session_id': session_id, |
| 'timestamp': datetime.now().isoformat(), |
| 'consent_given': True |
| } |
| |
| return depth_viz, plotly_fig, zip_path, report, json.dumps(metrics, indent=2) |
| |
| except Exception as e: |
| return None, None, None, f"Error: {str(e)}", None |
|
|
| |
| |
| |
|
|
| with gr.Blocks(title="Responsible AI 3D Reconstruction", theme=gr.themes.Soft()) as demo: |
| |
| gr.Markdown(""" |
| # 🏗️ 3D Reconstruction from Single Images |
| |
| |
| Transform 2D photographs into 3D spatial models |
| |
| <div style="background-color: #fff3cd; border: 2px solid #ffc107; padding: 15px; border-radius: 5px; margin: 10px 0;"> |
| <h3 style="color: #856404; margin-top: 0;">⚠️ Responsible Use Required</h3> |
| <p style="color: #856404; margin-bottom: 0;">This tool must be used ethically and legally. Review the guidelines in the <b>first tab</b>.</p> |
| </div> |
| """) |
| |
| with gr.Tabs(): |
| |
| with gr.Tab("⚠️ Responsible Use (READ FIRST)"): |
| gr.Markdown(RESPONSIBLE_AI_NOTICE) |
| gr.Markdown(""" |
| ### Known Limitations & Biases |
| - Trained primarily on Western indoor architecture |
| - May underperform on non-Western styles |
| - Scale is relative, not absolute |
| - Single viewpoint captures only visible surfaces |
| """) |
| |
| with gr.Tab("Reconstruction"): |
| consent_checkbox = gr.Checkbox( |
| label="**I have read and agree to the Responsible Use Guidelines**", |
| value=False |
| ) |
| |
| with gr.Row(): |
| with gr.Column(scale=1): |
| input_image = gr.Image( |
| type="pil", |
| label="Upload Image", |
| sources=["upload", "clipboard"] |
| ) |
| |
| model_choice = gr.Radio( |
| choices=["GLPN (Recommended)", "DPT (High Quality)"], |
| value="GLPN (Recommended)", |
| label="Depth Estimation Model" |
| ) |
| |
| visualization_type = gr.Radio( |
| choices=["mesh", "point_cloud"], |
| value="mesh", |
| label="Visualization Type" |
| ) |
| |
| reconstruct_btn = gr.Button("Start Reconstruction", variant="primary", size="lg") |
| |
| with gr.Column(scale=2): |
| depth_output = gr.Image(label="Depth Map") |
| viewer_3d = gr.Plot(label="Interactive 3D Viewer") |
| |
| with gr.Row(): |
| with gr.Column(): |
| metrics_output = gr.Markdown(label="Report") |
| with gr.Column(): |
| json_output = gr.Textbox(label="Metrics (JSON)", lines=8) |
| |
| download_output = gr.File(label="Download Package (ZIP)") |
| |
| reconstruct_btn.click( |
| fn=process_image_with_safeguards, |
| inputs=[input_image, model_choice, visualization_type, consent_checkbox], |
| outputs=[depth_output, viewer_3d, download_output, metrics_output, json_output] |
| ) |
| |
| with gr.Tab("Theory & Background"): |
| gr.Markdown(""" |
| ## About This Tool |
| |
| This application demonstrates how artificial intelligence can convert single 2D photographs |
| into interactive 3D models automatically. |
| |
| ### What Makes This Special |
| |
| **Traditional Approach:** |
| - Need special equipment (3D scanner, multiple cameras) |
| - Requires technical expertise |
| - Time-consuming process |
| - Expensive |
| |
| **This AI Approach:** |
| - Works with any single photograph |
| - No special equipment needed |
| - Automatic processing |
| - Free and accessible |
| |
| |
| |
| ## The Technology |
| |
| ### AI Model Used: GLPN |
| |
| **GLPN (Global-Local Path Networks)** |
| - Paper: Kim et al., CVPR 2022 |
| - Optimized for: Indoor/outdoor architectural scenes |
| - Training: NYU Depth V2 (urban indoor environments) |
| - Best for: Building interiors, street-level views |
| - Speed: Fast (~0.3-2.5s) |
| |
| ### How It Works (Simplified) |
| |
| 1. **AI analyzes photo** → Recognizes objects, patterns, perspective |
| 2. **Estimates distance** → Figures out what's close, what's far |
| 3. **Creates 3D points** → Places colored dots in 3D space |
| 4. **Builds surface** → Connects dots into smooth shape |
| |
| ### Spatial Data Pipeline |
| |
| **1. Monocular Depth Estimation** |
| - Challenge: Extracting 3D spatial information from 2D photographs |
| - Application: Similar to photogrammetry but from single images |
| - Output: Relative depth maps for spatial analysis |
| |
| **2. Point Cloud Generation** |
| - Creates 3D coordinate system (X, Y, Z) from pixels |
| - Each point: Spatial location + RGB color information |
| - Compatible with: GIS software, CAD tools, spatial databases |
| |
| **3. 3D Mesh Generation** |
| - Creates continuous surface from discrete points |
| - Similar to: Digital terrain models (DTMs) for buildings |
| - Output formats: Compatible with ArcGIS, QGIS, SketchUp |
| |
| ### Quality Metrics Explained |
| |
| - **Point Cloud Density**: Higher points = better spatial resolution |
| - **Geometric Accuracy**: Manifold checks ensure valid topology |
| - **Surface Continuity**: Watertight meshes = complete volume calculations |
| - **Data Fidelity**: Triangle count indicates level of detail |
| |
| ### Limitations for Geographic Applications |
| |
| 1. **Scale Ambiguity**: Requires ground control points for absolute measurements |
| 2. **Single Viewpoint**: Cannot capture occluded facades or hidden spaces |
| 3. **No Georeferencing**: Outputs in local coordinates, not global (lat/lon) |
| 4. **Weather Dependent**: Best results with clear, well-lit conditions |
| |
| ### Comparison with Traditional Methods |
| |
| **vs. Terrestrial Laser Scanning (TLS):** |
| - Much cheaper, faster, more accessible |
| - Lower accuracy, no absolute scale |
| |
| **vs. Photogrammetry (Structure-from-Motion):** |
| - Works with single image, faster processing |
| - Less accurate, cannot resolve scale |
| |
| **vs. LiDAR:** |
| - Much lower cost, consumer cameras sufficient |
| - Lower precision, no absolute measurements |
| |
| |
| |
| ## Reconstruction Pipeline (10 Steps) |
| |
| 1. **Image Preprocessing**: Resize to model requirements |
| 2. **Depth Estimation**: Neural network inference |
| 3. **Depth Visualization**: Create comparison images |
| 4. **Point Cloud Generation**: Back-project using camera model |
| 5. **Outlier Removal**: Statistical filtering |
| 6. **Normal Estimation**: Surface orientation calculation |
| 7. **Mesh Reconstruction**: Poisson surface reconstruction |
| 8. **Quality Metrics**: Compute geometric measures |
| 9. **3D Visualization**: Create interactive viewer |
| 10. **File Export**: Generate multiple formats |
| |
| ### Key References |
| |
| 1. **Kim, D., et al. (2022)**. "Global-Local Path Networks for Monocular Depth Estimation |
| with Vertical CutDepth." *CVPR 2022* |
| 2. **Kazhdan, M., et al. (2006)**. "Poisson Surface Reconstruction." |
| *Eurographics Symposium on Geometry Processing* |
| """) |
| |
| with gr.Tab("Usage Guide"): |
| gr.Markdown(""" |
| ## How to Use This Application |
| |
| ### Step 1: Read Responsible Use Guidelines |
| - **REQUIRED**: Review the "Responsible Use" tab first |
| - Understand privacy implications |
| - Acknowledge model limitations and biases |
| - Ensure you have rights to use source images |
| |
| ### Step 2: Prepare Your Image |
| |
| **Best Practices:** |
| - Remove EXIF metadata (GPS, timestamps) for privacy |
| - Ensure you have consent if image contains people |
| - Use well-lit, clear photographs |
| - Recommended resolution: 512-1024 pixels |
| - Indoor scenes work best |
| |
| **Privacy Checklist:** |
| - [ ] No identifiable people (or consent obtained) |
| - [ ] No sensitive/private locations |
| - [ ] EXIF data removed |
| - [ ] You own rights to the image |
| |
| ### Step 3: Upload Image |
| - Click "Upload Image" area |
| - Select JPG, PNG, or BMP file |
| - **Note:** Webcam option removed for privacy protection |
| - You can also paste from clipboard |
| |
| ### Step 4: Check Consent Box |
| - Check "I have read and agree to Responsible Use Guidelines" |
| - This confirms you've reviewed ethical guidelines |
| - Processing won't start without consent |
| |
| ### Step 5: Choose Visualization |
| - **Mesh**: Solid 3D surface (recommended) |
| - **Point Cloud**: Individual 3D points with colors |
| |
| ### Step 6: Start Reconstruction |
| - Click "Start Reconstruction" |
| - Processing takes 10-60 seconds |
| - All processing is local (no cloud upload) |
| |
| ### Step 7: Explore Results |
| |
| **Depth Map:** |
| - Yellow/Red = Farther objects |
| - Purple/Blue = Closer objects |
| - Shows AI's depth understanding |
| |
| **3D Viewer:** |
| - Rotate: Click and drag |
| - Zoom: Scroll wheel |
| - Pan: Right-click and drag |
| - Reset: Double-click |
| |
| **Metrics Report:** |
| - Processing performance |
| - Quality indicators |
| - Topology validation |
| |
| ### Step 8: Download Files |
| - ZIP package contains: |
| - Point cloud (PLY) |
| - Mesh (PLY, OBJ, STL) |
| - Quality metrics (JSON) |
| - All files include responsible AI metadata |
| |
| |
| |
| ## Viewing Downloaded 3D Files |
| |
| ### Free Software Options: |
| |
| **MeshLab** (Recommended for beginners) |
| - Download: https://www.meshlab.net/ |
| - Open PLY, OBJ, STL files |
| - Great for viewing and basic editing |
| |
| **Blender** (For advanced users) |
| - Download: https://www.blender.org/ |
| - Import → Wavefront (.obj) or PLY |
| - Full 3D modeling and rendering capabilities |
| |
| **CloudCompare** (For point clouds) |
| - Download: https://www.cloudcompare.org/ |
| - Best for analyzing point cloud data |
| - Measurement and analysis tools |
| |
| **Online Viewers** (No installation) |
| - https://3dviewer.net/ |
| - https://www.creators3d.com/online-viewer |
| - Just drag and drop your OBJ/PLY file |
| |
| |
| |
| |
| ## Tips for Best Results |
| |
| ### DO: |
| - Use well-lit images |
| - Include depth cues (corners, edges) |
| - Indoor scenes work best |
| - Medium resolution (512-1024px) |
| - Remove personal metadata |
| - Obtain consent for people in images |
| |
| ### AVOID: |
| - Motion blur or low resolution |
| - Reflective surfaces (mirrors, glass) |
| - Images without consent |
| - Private property without permission |
| - Surveillance or monitoring purposes |
| - Heavy shadows or darkness |
| |
| |
| ## Understanding the Metrics |
| |
| ### Point Cloud Statistics: |
| - **Initial Points**: Raw points generated from depth |
| - **Outliers Removed**: Noisy points filtered out (typically 5-15%) |
| - **Final Points**: Clean points used for mesh generation |
| |
| ### Mesh Quality Indicators: |
| - ** Edge Manifold**: Each edge connects exactly 2 faces (good topology) |
| - ** Vertex Manifold**: Clean vertex connections |
| - ** Watertight**: No holes, ready for 3D printing |
| - ** Marks**: Indicate potential issues (still usable, may need repair) |
| |
| ### Processing Times: |
| - **Depth Estimation**: 0.3-2.5s (GLPN model) |
| - **Mesh Reconstruction**: 2-10s (depends on point cloud size) |
| - **Total Time**: Usually 10-60 seconds |
| |
| --- |
| |
| ## Troubleshooting |
| |
| **Problem: No output appears** |
| - Check browser console for errors |
| - Try refreshing the page |
| - Try a smaller/simpler image first |
| - Check that image uploaded successfully |
| |
| **Problem: Mesh has holes or artifacts** |
| - This is normal for single-view reconstruction |
| - Hidden surfaces cannot be reconstructed |
| - Use mesh repair tools in MeshLab if needed |
| |
| **Problem: Colors look wrong on mesh** |
| - Vertex color interpolation is approximate |
| - This is expected behavior |
| - Colors on point cloud are more accurate |
| |
| **Problem: Processing is very slow** |
| - Use smaller images |
| - This is normal on CPU (GPU is much faster) |
| |
| **Problem: "Not watertight" in metrics** |
| - Common for complex scenes |
| - Still usable for visualization |
| - For 3D printing: use mesh repair in MeshLab |
| """) |
| |
| with gr.Tab(" Ethics & Impact"): |
| gr.Markdown(""" |
| |
| ## Algorithmic Bias & Fairness |
| |
| ### Training Data Representation |
| |
| **Geographic Bias:** |
| - Heavy representation: North America, Europe |
| - Underrepresented: Africa, South Asia, Pacific Islands |
| - Impact: Lower accuracy for non-Western architecture |
| |
| **Architectural Style Bias:** |
| - Well-represented: Modern interiors, Western buildings |
| - Underrepresented: Traditional, vernacular, indigenous structures |
| - Impact: May misinterpret non-standard spatial layouts |
| |
| **Socioeconomic Bias:** |
| - Training data skewed toward middle/upper-class interiors |
| - Limited representation of informal settlements |
| - May not generalize well to all socioeconomic contexts |
| |
| |
| |
| |
| |
| ### Potential Harms |
| |
| ** Privacy Violations:** |
| - Unauthorized 3D reconstruction of private spaces |
| - Creating models of individuals without consent |
| - Surveillance and tracking applications |
| |
| ** Misinformation:** |
| - Generating fake 3D evidence |
| - Manipulating spatial understanding |
| - Creating misleading visualizations |
| |
| ** Property Rights:** |
| - Unauthorized documentation of copyrighted designs |
| - Intellectual property theft |
| - Commercial exploitation without permission |
| |
| ### Harm Prevention |
| |
| 1. **Mandatory consent**: Require user acknowledgment |
| 2. **Use case restriction**: Prohibit surveillance and deceptive uses |
| 3. **Privacy protection**: Disable webcam, encourage EXIF removal |
| 4. **Transparency**: Clear documentation of limitations |
| |
| |
| |
| ## Accountability & Governance |
| |
| ### User Responsibilities |
| |
| As a user, you are responsible for: |
| - Ensuring lawful use of source images |
| - Obtaining necessary consents and permissions |
| - Respecting privacy and intellectual property |
| - Using outputs ethically and transparently |
| - Understanding and accounting for model biases |
| |
| ### Developer Responsibilities |
| |
| This tool implements: |
| - Clear responsible use guidelines |
| - Privacy-protective design (no webcam, local processing) |
| - Bias documentation and transparency |
| - Prohibited use cases explicitly stated |
| |
| |
| ## Future Directions |
| |
| ### Improving Fairness |
| - Train on more diverse geographic datasets |
| - Include underrepresented architectural styles |
| - Develop bias mitigation techniques |
| - Community-driven model evaluation |
| |
| ### Enhancing Privacy |
| - Face/person detection and redaction |
| - Automatic EXIF stripping |
| - Differential privacy techniques |
| """) |
| |
| with gr.Tab(" Citation"): |
| gr.Markdown(""" |
| ## Academic Citation |
| |
| ### For GLPN Model: |
| ```bibtex |
| @inproceedings{kim2022global, |
| title={Global-Local Path Networks for Monocular Depth Estimation with Vertical CutDepth}, |
| author={Kim, Doyeon and Ga, Woonghyun and Ahn, Pyungwhan and Joo, Donggyu and Chun, Sehwan and Kim, Junmo}, |
| booktitle={CVPR}, |
| year={2022} |
| } |
| ``` |
| |
| ### For Poisson Surface Reconstruction: |
| ```bibtex |
| @inproceedings{kazhdan2006poisson, |
| title={Poisson Surface Reconstruction}, |
| author={Kazhdan, Michael and Bolitho, Matthew and Hoppe, Hugues}, |
| booktitle={Symposium on Geometry Processing}, |
| year={2006} |
| } |
| ``` |
| |
| ## Open Source Components |
| |
| This application is built with: |
| |
| - **Transformers** (Hugging Face): Model inference framework |
| - **Open3D**: Point cloud and mesh processing |
| - **PyTorch**: Deep learning framework |
| - **Plotly**: Interactive 3D visualization |
| - **Gradio**: Web interface framework |
| - **NumPy** & **SciPy**: Numerical computing |
| - **Matplotlib**: Data visualization |
| - **Pillow (PIL)**: Image processing |
| |
| ## Model Credits |
| |
| **GLPN Model:** |
| - Developed by: KAIST (Korea Advanced Institute of Science and Technology) |
| - Hosted by: Hugging Face (vinvino02/glpn-nyu) |
| - License: Apache 2.0 |
| |
| ## Responsible AI Features |
| |
| This implementation includes: |
| - Privacy-protective design (no webcam option) |
| - Mandatory consent acknowledgment |
| - Bias documentation and transparency |
| - Ethical use guidelines |
| |
| |
| |
| """) |
| |
| gr.Markdown(""" |
| --- |
| |
| **Version:** 2.0 (Responsible AI Edition - Optimized) |
| **Last Updated:** 2025 |
| **License:** Educational and Research Use |
| |
| """) |
|
|
| if __name__ == "__main__": |
| print("="*60) |
| print("RESPONSIBLE AI 3D RECONSTRUCTION") |
| print("="*60) |
| print("✓ Lightweight model (GLPN only)") |
| print("✓ No webcam option") |
| print("✓ Local processing") |
| print("✓ Consent required") |
| print("="*60) |
| demo.launch(share=True) |
