From 33301a35c3eefc50c7386ebcb3685acf3a3be6d1 Mon Sep 17 00:00:00 2001 From: tobiaobr Date: Mon, 3 Nov 2025 15:42:20 +0100 Subject: [PATCH] updated docs --- README.MD | 61 +++++++-- conversion_log.txt | 330 --------------------------------------------- run.py | 31 ----- run.sh | 2 +- 4 files changed, 50 insertions(+), 374 deletions(-) delete mode 100644 conversion_log.txt delete mode 100644 run.py diff --git a/README.MD b/README.MD index c892b34..fc3811a 100644 --- a/README.MD +++ b/README.MD @@ -7,9 +7,11 @@ The main steps are: 2. Convert CSV Surveys to LAS using EPSG:4978 3. Group and Crop by H3 Cells -# How to run +`./PotreeConverter` converts LAS files into Potree's Octree Structure, which the Molloy Explorer Potree Viewer supports as its native input format. -## 1. Prepare Files and Folders +## How to run + +### 1. Prepare Files and Folders Before running the script, make sure your project folder has the following structure: @@ -27,39 +29,44 @@ metadata/ └── level1_h3_cells.csv ``` -### `surveys/` +#### `surveys/` + - Contains **one folder per survey**, using the **survey name** as the folder name. - Each survey folder contains the CSV files for that survey (arbitrarily named). -### `metadata/metadata.csv` +#### `metadata/metadata.csv` + - Must include the columns: `fid`, `survey_name`, `survey_area`, `geom`, `epsg`. - Script assumes bounding polygons in `geom` use EPSG:4258. - Each row corresponds to a survey. -### `metadata/level1_h3_cells.csv` -- Must contain a single column `cell_id`, listing all relevant **level 1 H3 cells**. +#### `metadata/level1_h3_cells.csv` +- Must contain a single column `cell_id`, listing all relevant **level 1 H3 cells**. -## 2. Activate python environment +### 2. Activate python environment Create a python virtual environment with + ```zsh python -m venv ``` Activate it: -* **Windows:** +- **Windows:** + ```zsh \Scripts\activate ``` -* **macOS/Linux:** +- **macOS/Linux:** + ```zsh source /bin/activate ``` -## 3. Install requirements +### 3. Install requirements Install the required Python packages: @@ -67,8 +74,7 @@ Install the required Python packages: pip install -r requirements.txt ``` - -## 4. Run the script +### 4. Run the lasConverter script Run the converter: @@ -77,3 +83,34 @@ python lasConverter.py ``` The processed LAS files, cropped to level 2 H3 cells, will be saved in the `./h3_cells` folder. + +#### Setup PotreeConvert + +First make a build directory in the Potree directory + +```zsh +cd PotreeConverter +mkdir build +cd build +``` + +Then run + +```zsh +cmake ../ +make +``` + +### 6. Run the PotreeConverter + +From the root of the repository, run the run.sh script to convert the H3 cells into the Potree Structure. + +```zsh +./run.sh +``` + +The output is saved in `./pointclouds`. It can then be directly moved into the Viewer or the shell script can be adjusted to automatically save the files in the Viewer. + +### 7. Optimizations + +This script runs the PotreeConverter on each of the H3 cells sequentially. It is possible to modify the shell script to run these conversions in parallell. diff --git a/conversion_log.txt b/conversion_log.txt deleted file mode 100644 index df42e8a..0000000 --- a/conversion_log.txt +++ /dev/null @@ -1,330 +0,0 @@ -[14:15:20] Converting converted_surveys/dof-2019-nh03-b02.las -> pointclouds/cell_1 -#threads: 16 -#paths: 1 - -output attributes: -name offset size -================================================ -position 0 12 -intensity 12 2 -return number 14 1 -number of returns 15 1 -classification flags 16 1 -classification 17 1 -user data 18 1 -scan angle 19 2 -point source id 21 2 -gps-time 23 8 -Accepted 31 1 -TVU 32 4 -THU 36 4 -================================================ - 40 -================================================ -cubicAABB: { - "min": [2097901.740000, 264150.350000, 5976616.220000], - "max": [2146045.070000, 312293.680000, 6024759.550000], - "size": [48143.330000, 48143.330000, 48143.330000] -} -#points: 7'078'190 -total file size: 263.3 MB -target directory: '/home/tobias/Documents/molloyexplorer/lasConverter/pointclouds/cell_1' - -======================================= -=== COUNTING -======================================= -tStartTaskAssembly: 0.000099s -countPointsInCells: 0.202839s -finished counting in 0s -======================================= -createLUT: 0.025373s - -======================================= -=== CREATING CHUNKS -======================================= -distributePoints0: 0.000266s -distributePoints1: 0.000269s -finished creating chunks in 0s -======================================= - -======================================= -=== INDEXING -======================================= - -[67%, 1s], [INDEXING: 0%, duration: 0s, throughput: -nanMPs][RAM: 2.6GB (highest 2.8GB), CPU: 94%] -sampling: 0.672293s -flushing: 0.768599s -metadata & hierarchy: 0.782649s -deleting temporary files - -======================================= -=== STATS -======================================= -#points: 7'078'190 -#input files: 1 -sampling method: poisson -chunk method: LASZIP -input file size: 0.3GB -duration: 2.008s -throughput (MB/s) 131MB -throughput (points/s) 3.5M -output location: /home/tobias/Documents/molloyexplorer/lasConverter/pointclouds/cell_1 -duration(chunking-count): 0.203 -duration(chunking-distribute): 0.330 -duration(chunking-total): 0.535 -duration(indexing): 0.785 -[14:15:22] ✅ Success: converted_surveys/dof-2019-nh03-b02.las -[14:15:22] Converting converted_surveys/dof-2019-nh06-b01.las -> pointclouds/cell_2 -#threads: 16 -#paths: 1 - -output attributes: -name offset size -================================================ -position 0 12 -intensity 12 2 -return number 14 1 -number of returns 15 1 -classification flags 16 1 -classification 17 1 -user data 18 1 -scan angle 19 2 -point source id 21 2 -gps-time 23 8 -Accepted 31 1 -TVU 32 4 -THU 36 4 -================================================ - 40 -================================================ -cubicAABB: { - "min": [1110737.710000, 30470.990000, 6234599.770000], - "max": [1230011.770000, 149745.050000, 6353873.830000], - "size": [119274.060000, 119274.060000, 119274.060000] -} -#points: 38'002'068 -total file size: 1.4 GB -target directory: '/home/tobias/Documents/molloyexplorer/lasConverter/pointclouds/cell_2' - -======================================= -=== COUNTING -======================================= -tStartTaskAssembly: 0.000035s -countPointsInCells: 0.612417s -finished counting in 1s -======================================= -createLUT: 0.025339s - -======================================= -=== CREATING CHUNKS -======================================= -distributePoints0: 0.000267s -distributePoints1: 0.000270s - -[33%, 1s], [DISTRIBUTING: 0%, duration: 0s, throughput: -nanMPs][RAM: 2.0GB (highest 2.0GB), CPU: 78%] -finished creating chunks in 1s -======================================= - -======================================= -=== INDEXING -======================================= -[67%, 2s], [INDEXING: 0%, duration: 0s, throughput: -nanMPs][RAM: 2.2GB (highest 3.1GB), CPU: 90%] -[67%, 3s], [INDEXING: 0%, duration: 0s, throughput: -nanMPs][RAM: 4.2GB (highest 4.2GB), CPU: 97%] -[73%, 4s], [INDEXING: 18%, duration: 1s, throughput: 7MPs][RAM: 4.6GB (highest 4.6GB), CPU: 98%] -[81%, 5s], [INDEXING: 42%, duration: 2s, throughput: 7MPs][RAM: 4.0GB (highest 4.6GB), CPU: 92%] -sampling: 3.381163s -flushing: 3.951287s -metadata & hierarchy: 3.962745s -deleting temporary files - -======================================= -=== STATS -======================================= -#points: 38'002'068 -#input files: 1 -sampling method: poisson -chunk method: LASZIP -input file size: 1.4GB -duration: 6.015s -throughput (MB/s) 235MB -throughput (points/s) 6.3M -output location: /home/tobias/Documents/molloyexplorer/lasConverter/pointclouds/cell_2 -duration(chunking-count): 0.612 -duration(chunking-distribute): 1.343 -duration(chunking-total): 1.997 -duration(indexing): 3.964 -[14:15:28] ✅ Success: converted_surveys/dof-2019-nh06-b01.las -[14:15:28] Converting converted_surveys/franklin-2011-111.las -> pointclouds/cell_3 -#threads: 16 -#paths: 1 - -output attributes: -name offset size -================================================ -position 0 12 -intensity 12 2 -return number 14 1 -number of returns 15 1 -classification flags 16 1 -classification 17 1 -user data 18 1 -scan angle 19 2 -point source id 21 2 -gps-time 23 8 -Accepted 31 1 -TVU 32 4 -THU 36 4 -================================================ - 40 -================================================ -cubicAABB: { - "min": [1638485.370000, 1063748.890000, 6039093.650000], - "max": [1670168.310000, 1095431.830000, 6070776.590000], - "size": [31682.940000, 31682.940000, 31682.940000] -} -#points: 413'146'577 -total file size: 15.0 GB -target directory: '/home/tobias/Documents/molloyexplorer/lasConverter/pointclouds/cell_3' - -======================================= -=== COUNTING -======================================= -tStartTaskAssembly: 0.000097s - -[5%, 1s], [COUNTING: 15%, duration: 1s, throughput: 70MPs][RAM: 1.9GB (highest 1.9GB), CPU: 88%] -[11%, 2s], [COUNTING: 34%, duration: 2s, throughput: 72MPs][RAM: 1.9GB (highest 1.9GB), CPU: 97%] -[17%, 3s], [COUNTING: 52%, duration: 3s, throughput: 72MPs][RAM: 1.9GB (highest 1.9GB), CPU: 96%] -[23%, 4s], [COUNTING: 69%, duration: 4s, throughput: 72MPs][RAM: 1.9GB (highest 1.9GB), CPU: 95%] -[29%, 5s], [COUNTING: 87%, duration: 5s, throughput: 73MPs][RAM: 1.9GB (highest 1.9GB), CPU: 97%] -countPointsInCells: 5.638949s -finished counting in 6s -======================================= -createLUT: 0.252734s - -======================================= -=== CREATING CHUNKS -======================================= -distributePoints0: 0.000266s -distributePoints1: 0.000270s -[33%, 6s], [DISTRIBUTING: 0%, duration: 0s, throughput: -nanMPs][RAM: 2.1GB (highest 2.1GB), CPU: 63%] -[35%, 7s], [DISTRIBUTING: 6%, duration: 1s, throughput: 25MPs][RAM: 3.3GB (highest 3.5GB), CPU: 96%] -[38%, 8s], [DISTRIBUTING: 13%, duration: 2s, throughput: 25MPs][RAM: 3.4GB (highest 3.5GB), CPU: 75%] -[40%, 9s], [DISTRIBUTING: 19%, duration: 3s, throughput: 26MPs][RAM: 3.6GB (highest 3.6GB), CPU: 76%] -[42%, 10s], [DISTRIBUTING: 25%, duration: 4s, throughput: 25MPs][RAM: 3.9GB (highest 3.9GB), CPU: 76%] -[44%, 11s], [DISTRIBUTING: 31%, duration: 5s, throughput: 25MPs][RAM: 4.4GB (highest 4.4GB), CPU: 77%] -[45%, 12s], [DISTRIBUTING: 36%, duration: 6s, throughput: 25MPs][RAM: 4.7GB (highest 4.7GB), CPU: 68%] -[47%, 13s], [DISTRIBUTING: 42%, duration: 7s, throughput: 25MPs][RAM: 5.6GB (highest 5.6GB), CPU: 72%] -[49%, 14s], [DISTRIBUTING: 48%, duration: 8s, throughput: 25MPs][RAM: 5.9GB (highest 5.9GB), CPU: 72%] -[50%, 15s], [DISTRIBUTING: 49%, duration: 8s, throughput: 24MPs][RAM: 5.7GB (highest 6.0GB), CPU: 56%] -[52%, 16s], [DISTRIBUTING: 55%, duration: 10s, throughput: 23MPs][RAM: 6.0GB (highest 6.0GB), CPU: 71%] -[53%, 17s], [DISTRIBUTING: 59%, duration: 11s, throughput: 22MPs][RAM: 6.3GB (highest 6.4GB), CPU: 52%] -[54%, 18s], [DISTRIBUTING: 63%, duration: 12s, throughput: 22MPs][RAM: 6.4GB (highest 6.5GB), CPU: 91%] -[56%, 19s], [DISTRIBUTING: 69%, duration: 13s, throughput: 22MPs][RAM: 6.3GB (highest 6.7GB), CPU: 97%] -[58%, 20s], [DISTRIBUTING: 74%, duration: 14s, throughput: 22MPs][RAM: 6.2GB (highest 6.7GB), CPU: 93%] -[59%, 21s], [DISTRIBUTING: 78%, duration: 15s, throughput: 22MPs][RAM: 6.3GB (highest 6.7GB), CPU: 71%] -[61%, 22s], [DISTRIBUTING: 82%, duration: 16s, throughput: 22MPs][RAM: 6.4GB (highest 6.7GB), CPU: 86%] -[63%, 23s], [DISTRIBUTING: 89%, duration: 17s, throughput: 22MPs][RAM: 6.4GB (highest 6.7GB), CPU: 82%] -[64%, 24s], [DISTRIBUTING: 93%, duration: 18s, throughput: 22MPs][RAM: 6.6GB (highest 6.7GB), CPU: 68%] -[66%, 25s], [DISTRIBUTING: 97%, duration: 19s, throughput: 21MPs][RAM: 6.4GB (highest 6.7GB), CPU: 76%] -[67%, 26s], [DISTRIBUTING: 100%, duration: 20s, throughput: 21MPs][RAM: 5.7GB (highest 6.7GB), CPU: 53%] -[67%, 27s], [DISTRIBUTING: 100%, duration: 20s, throughput: 21MPs][RAM: 5.5GB (highest 6.7GB), CPU: 36%] -finished creating chunks in 22s -======================================= -[67%, 28s], [DISTRIBUTING: 100%, duration: 20s, throughput: 21MPs][RAM: 2.2GB (highest 6.7GB), CPU: 25%] - -======================================= -=== INDEXING -======================================= -[67%, 29s], [INDEXING: 0%, duration: 0s, throughput: -nanMPs][RAM: 5.8GB (highest 6.7GB), CPU: 77%] -[67%, 30s], [INDEXING: 0%, duration: 0s, throughput: -nanMPs][RAM: 6.7GB (highest 6.7GB), CPU: 94%] -[67%, 31s], [INDEXING: 1%, duration: 2s, throughput: 1MPs][RAM: 7.2GB (highest 7.2GB), CPU: 94%] -[67%, 32s], [INDEXING: 2%, duration: 3s, throughput: 3MPs][RAM: 8.1GB (highest 8.1GB), CPU: 98%] -[67%, 34s], [INDEXING: 2%, duration: 3s, throughput: 3MPs][RAM: 8.5GB (highest 8.5GB), CPU: 96%] -[67%, 38s], [INDEXING: 2%, duration: 3s, throughput: 3MPs][RAM: 8.4GB (highest 8.5GB), CPU: 99%] -[14:16:32] ❌ Failed: converted_surveys/franklin-2011-111.las -[14:16:32] Converting converted_surveys/sverdrup-2019-027-m.las -> pointclouds/cell_4 -#threads: 16 -#paths: 1 - -output attributes: -name offset size -================================================ -position 0 12 -intensity 12 2 -return number 14 1 -number of returns 15 1 -classification flags 16 1 -classification 17 1 -user data 18 1 -scan angle 19 2 -point source id 21 2 -gps-time 23 8 -Accepted 31 1 -TVU 32 4 -THU 36 4 -================================================ - 40 -================================================ -cubicAABB: { - "min": [1996419.580000, -472997.980000, 6006612.020000], - "max": [2040314.020000, -429103.540000, 6050506.460000], - "size": [43894.440000, 43894.440000, 43894.440000] -} -#points: 347'632'251 -total file size: 12.6 GB -target directory: '/home/tobias/Documents/molloyexplorer/lasConverter/pointclouds/cell_4' - -======================================= -=== COUNTING -======================================= -tStartTaskAssembly: 0.000180s - -[4%, 1s], [COUNTING: 11%, duration: 1s, throughput: 43MPs][RAM: 1.9GB (highest 1.9GB), CPU: 99%] -[9%, 2s], [COUNTING: 26%, duration: 2s, throughput: 48MPs][RAM: 1.9GB (highest 1.9GB), CPU: 93%] -[16%, 3s], [COUNTING: 47%, duration: 3s, throughput: 57MPs][RAM: 1.9GB (highest 1.9GB), CPU: 95%] -[23%, 4s], [COUNTING: 68%, duration: 4s, throughput: 61MPs][RAM: 1.9GB (highest 1.9GB), CPU: 96%] -[29%, 5s], [COUNTING: 88%, duration: 5s, throughput: 63MPs][RAM: 1.9GB (highest 1.9GB), CPU: 97%] -countPointsInCells: 5.383764s -finished counting in 5s -======================================= -createLUT: 0.239510s - -======================================= -=== CREATING CHUNKS -======================================= -distributePoints0: 0.000256s -distributePoints1: 0.000261s -[33%, 6s], [DISTRIBUTING: 0%, duration: 0s, throughput: -nanMPs][RAM: 2.1GB (highest 2.1GB), CPU: 68%] -[36%, 7s], [DISTRIBUTING: 9%, duration: 1s, throughput: 27MPs][RAM: 3.2GB (highest 3.5GB), CPU: 96%] -[39%, 8s], [DISTRIBUTING: 17%, duration: 2s, throughput: 27MPs][RAM: 3.4GB (highest 3.5GB), CPU: 78%] -[42%, 9s], [DISTRIBUTING: 26%, duration: 3s, throughput: 28MPs][RAM: 3.8GB (highest 3.8GB), CPU: 91%] -[44%, 10s], [DISTRIBUTING: 32%, duration: 4s, throughput: 27MPs][RAM: 3.4GB (highest 3.8GB), CPU: 77%] -[47%, 11s], [DISTRIBUTING: 40%, duration: 5s, throughput: 27MPs][RAM: 4.0GB (highest 4.0GB), CPU: 85%] -[49%, 12s], [DISTRIBUTING: 47%, duration: 6s, throughput: 26MPs][RAM: 4.2GB (highest 4.2GB), CPU: 76%] -[51%, 13s], [DISTRIBUTING: 54%, duration: 7s, throughput: 26MPs][RAM: 4.7GB (highest 4.7GB), CPU: 82%] -[54%, 14s], [DISTRIBUTING: 61%, duration: 8s, throughput: 26MPs][RAM: 5.1GB (highest 5.1GB), CPU: 83%] -[56%, 15s], [DISTRIBUTING: 68%, duration: 9s, throughput: 26MPs][RAM: 5.4GB (highest 5.4GB), CPU: 79%] -[57%, 16s], [DISTRIBUTING: 72%, duration: 10s, throughput: 26MPs][RAM: 5.4GB (highest 5.5GB), CPU: 63%] -[60%, 17s], [DISTRIBUTING: 80%, duration: 11s, throughput: 25MPs][RAM: 5.8GB (highest 5.8GB), CPU: 76%] -[61%, 18s], [DISTRIBUTING: 82%, duration: 12s, throughput: 24MPs][RAM: 5.7GB (highest 5.8GB), CPU: 46%] -[62%, 19s], [DISTRIBUTING: 87%, duration: 13s, throughput: 23MPs][RAM: 5.8GB (highest 5.9GB), CPU: 79%] -[64%, 20s], [DISTRIBUTING: 93%, duration: 14s, throughput: 23MPs][RAM: 5.9GB (highest 5.9GB), CPU: 77%] -[66%, 21s], [DISTRIBUTING: 99%, duration: 15s, throughput: 23MPs][RAM: 5.9GB (highest 6.2GB), CPU: 56%] -[67%, 22s], [DISTRIBUTING: 100%, duration: 16s, throughput: 22MPs][RAM: 5.3GB (highest 6.2GB), CPU: 34%] -[67%, 23s], [DISTRIBUTING: 100%, duration: 16s, throughput: 22MPs][RAM: 4.9GB (highest 6.2GB), CPU: 26%] -finished creating chunks in 18s -======================================= - -======================================= -=== INDEXING -======================================= -[67%, 24s], [INDEXING: 0%, duration: 0s, throughput: -nanMPs][RAM: 5.0GB (highest 6.2GB), CPU: 21%] -[67%, 25s], [INDEXING: 0%, duration: 1s, throughput: 1MPs][RAM: 5.9GB (highest 6.2GB), CPU: 82%] -[67%, 26s], [INDEXING: 2%, duration: 2s, throughput: 3MPs][RAM: 7.1GB (highest 7.1GB), CPU: 94%] -[67%, 27s], [INDEXING: 2%, duration: 2s, throughput: 3MPs][RAM: 7.6GB (highest 7.6GB), CPU: 98%] -[14:18:22] ❌ Failed: converted_surveys/sverdrup-2019-027-m.las --------------------------------------------------- -Conversion finished. Summary: -Total files processed: 4 -✅ Successful: 2 -❌ Failed: 2 diff --git a/run.py b/run.py deleted file mode 100644 index 4b64142..0000000 --- a/run.py +++ /dev/null @@ -1,31 +0,0 @@ -import os -import subprocess - -input_dir = "converted_surveys" -output_base = "pointclouds" -converter_path = "./PotreeConverter/build/PotreeConverter" - -las_files = [f for f in os.listdir(input_dir) if f.endswith(".las")] - -failed = [] - -for i, las_file in enumerate(las_files, start=1): - input_path = os.path.join(input_dir, las_file) - output_dir = os.path.join(output_base, f"cell_{i}") - os.makedirs(output_dir, exist_ok=True) - - cmd = [converter_path, input_path, "-o", output_dir] - - print(f"Converting {las_file} -> {output_dir}") - try: - subprocess.run(cmd, check=True) - except subprocess.CalledProcessError as e: - print(f"❌ Conversion failed for {las_file}: {e}") - failed.append(las_file) - continue - -print("✅ Conversion finished.") -if failed: - print("⚠️ The following files failed to convert:") - for f in failed: - print(f" - {f}") diff --git a/run.sh b/run.sh index 6cce9ba..2693ccd 100755 --- a/run.sh +++ b/run.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -u # Treat unset vars as errors -input_dir="converted_surveys" +input_dir="h3_cells" output_base="pointclouds" converter="./PotreeConverter/build/PotreeConverter" logfile="conversion_log.txt"