diff --git a/src/main.rs b/src/main.rs index 8931861..8bd5919 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,7 +5,9 @@ mod styles; use crate::{ questions::{question_ckan_version, question_ssh, question_sysadmin}, steps::{ - step_install_ahoy, step_install_and_run_ckan_compose, step_install_curl, + step_install_ahoy, step_install_and_run_ckan_compose, + step_install_ckanext_scheming_extension, step_install_curl, + step_install_datapusher_plus_extension, step_install_datastore_extension, step_install_docker, step_install_openssh, step_package_updates, }, styles::{important_text, step_text, success_text}, @@ -14,7 +16,6 @@ use anyhow::Result; use clap::{Parser, Subcommand}; use human_panic::{metadata, setup_panic}; use inquire::Confirm; -use serde_json::json; use std::{path::PathBuf, str::FromStr}; use xshell::cmd; use xshell_venv::{Shell, VirtualEnv}; @@ -263,203 +264,20 @@ rm -rf README ckan-compose ahoy dpp_default_config.ini get-docker.sh permissions success_text(format!("6. Installed CKAN {}.", config.ckan_version).as_str()) ); + // Install extensions + if config.extension_datastore { + step_install_datastore_extension("7.".to_string(), &sh, username.clone())?; + } + if config.extension_ckanext_scheming { + step_install_ckanext_scheming_extension("8.".to_string(), &sh, username.clone())?; + } if config.extension_datapusher_plus { - println!( - "\n{} Enabling DataStore plugin, adding config URLs in /etc/ckan/default/ckan.ini and updating permissions...", - step_text("7."), - ); - let mut conf = ini::Ini::load_from_file("/etc/ckan/default/ckan.ini")?; - let app_main_section = conf.section_mut(Some("app:main")).unwrap(); - let mut ckan_plugins = app_main_section.get("ckan.plugins").unwrap().to_string(); - ckan_plugins.push_str(" datastore"); - app_main_section.insert("ckan.plugins", ckan_plugins); - app_main_section.insert( - "ckan.datastore.write_url", - "postgresql://ckan_default:pass@localhost/datastore_default", - ); - app_main_section.insert( - "ckan.datastore.read_url", - "postgresql://datastore_default:pass@localhost/datastore_default", - ); - app_main_section.insert("ckan.datastore.sqlsearch.enabled", "true"); - conf.write_to_file("/etc/ckan/default/ckan.ini")?; - let postgres_container_id = cmd!( - sh, - "sudo docker ps -aqf name=^ckan-devstaller-project-postgres$" - ) - .read()?; - let set_permissions_output = cmd!( - sh, - "ckan -c /etc/ckan/default/ckan.ini datastore set-permissions" - ) - .read()?; - std::fs::write("permissions.sql", set_permissions_output)?; - loop { - std::thread::sleep(std::time::Duration::from_secs(2)); - if std::fs::exists("permissions.sql")? { - break; - } - } - sh.change_dir(format!("/home/{username}")); - cmd!( - sh, - "sudo docker cp permissions.sql {postgres_container_id}:/permissions.sql" - ) - .run()?; - cmd!(sh, "sudo docker exec {postgres_container_id} psql -U ckan_default --set ON_ERROR_STOP=1 -f permissions.sql").run()?; - println!( - "{}", - success_text( - "7. Enabled DataStore plugin, set DataStore URLs in /etc/ckan/default/ckan.ini, and updated permissions." - ) - ); - - println!( - "{}", - step_text("\n{} Installing ckanext-scheming and DataPusher+ extensions..."), - ); - cmd!( - sh, - "pip install -e git+https://github.com/ckan/ckanext-scheming.git#egg=ckanext-scheming" - ) - .run()?; - let mut conf = ini::Ini::load_from_file("/etc/ckan/default/ckan.ini")?; - let app_main_section = conf.section_mut(Some("app:main")).unwrap(); - let mut ckan_plugins = app_main_section.get("ckan.plugins").unwrap().to_string(); - ckan_plugins.push_str(" scheming_datasets"); - cmd!( - sh, - "ckan config-tool /etc/ckan/default/ckan.ini -s app:main ckan.plugins={ckan_plugins}" - ) - .run()?; - cmd!(sh, "ckan config-tool /etc/ckan/default/ckan.ini -s app:main scheming.presets=ckanext.scheming:presets.json").run()?; - cmd!(sh, "ckan config-tool /etc/ckan/default/ckan.ini -s app:main scheming.dataset_fallback=false").run()?; - // app_main_section.insert("ckan.plugins", ckan_plugins); - // app_main_section.insert("scheming.presets", "ckanext.scheming:presets.json"); - // app_main_section.insert("scheming.dataset_fallback", "false"); - // conf.write_to_file("/etc/ckan/default/ckan.ini")?; - // Install DataPusher+ - cmd!(sh, "sudo apt install python3-virtualenv python3-dev python3-pip python3-wheel build-essential libxslt1-dev libxml2-dev zlib1g-dev git libffi-dev libpq-dev uchardet -y").run()?; - sh.change_dir("/usr/lib/ckan/default/src"); - cmd!(sh, "pip install -e git+https://github.com/dathere/datapusher-plus.git@main#egg=datapusher-plus").run()?; - sh.change_dir("/usr/lib/ckan/default/src/datapusher-plus"); - cmd!(sh, "pip install -r requirements.txt").run()?; - sh.change_dir(format!("/home/{username}")); - cmd!(sh, "wget https://github.com/dathere/qsv/releases/download/4.0.0/qsv-4.0.0-x86_64-unknown-linux-gnu.zip").run()?; - cmd!(sh, "sudo apt install unzip -y").run()?; - cmd!(sh, "unzip qsv-4.0.0-x86_64-unknown-linux-gnu.zip").run()?; - cmd!(sh, "sudo rm -rf qsv-4.0.0-x86_64-unknown-linux-gnu.zip").run()?; - cmd!(sh, "sudo mv ./qsvdp_glibc-2.31 /usr/local/bin/qsvdp").run()?; - let mut conf = ini::Ini::load_from_file("/etc/ckan/default/ckan.ini")?; - let app_main_section = conf.section_mut(Some("app:main")).unwrap(); - let mut ckan_plugins = app_main_section.get("ckan.plugins").unwrap().to_string(); - ckan_plugins.push_str(" datapusher_plus"); - cmd!( - sh, - "ckan config-tool /etc/ckan/default/ckan.ini -s app:main ckan.plugins={ckan_plugins}" - ) - .run()?; - cmd!(sh, "ckan config-tool /etc/ckan/default/ckan.ini -s app:main scheming.dataset_schemas=ckanext.datapusher_plus:dataset-druf.yaml").run()?; - // app_main_section.insert("ckan.plugins", ckan_plugins); - // app_main_section.insert( - // "scheming.dataset_schemas", - // "ckanext.datapusher_plus:dataset-druf.yaml", - // ); - // conf.write_to_file("/etc/ckan/default/ckan.ini")?; - let dpp_default_config = r#" -ckanext.datapusher_plus.use_proxy = false -ckanext.datapusher_plus.download_proxy = -ckanext.datapusher_plus.ssl_verify = false -# supports INFO, DEBUG, TRACE - use DEBUG or TRACE when debugging scheming Formulas -ckanext.datapusher_plus.upload_log_level = INFO -ckanext.datapusher_plus.formats = csv tsv tab ssv xls xlsx xlsxb xlsm ods geojson shp qgis zip -ckanext.datapusher_plus.pii_screening = false -ckanext.datapusher_plus.pii_found_abort = false -ckanext.datapusher_plus.pii_regex_resource_id_or_alias = -ckanext.datapusher_plus.pii_show_candidates = false -ckanext.datapusher_plus.pii_quick_screen = false -ckanext.datapusher_plus.qsv_bin = /usr/local/bin/qsvdp -ckanext.datapusher_plus.preview_rows = 100 -ckanext.datapusher_plus.download_timeout = 300 -ckanext.datapusher_plus.max_content_length = 1256000000000 -ckanext.datapusher_plus.chunk_size = 16384 -ckanext.datapusher_plus.default_excel_sheet = 0 -ckanext.datapusher_plus.sort_and_dupe_check = true -ckanext.datapusher_plus.dedup = false -ckanext.datapusher_plus.unsafe_prefix = unsafe_ -ckanext.datapusher_plus.reserved_colnames = _id -ckanext.datapusher_plus.prefer_dmy = false -ckanext.datapusher_plus.ignore_file_hash = true -ckanext.datapusher_plus.auto_index_threshold = 3 -ckanext.datapusher_plus.auto_index_dates = true -ckanext.datapusher_plus.auto_unique_index = true -ckanext.datapusher_plus.summary_stats_options = -ckanext.datapusher_plus.add_summary_stats_resource = false -ckanext.datapusher_plus.summary_stats_with_preview = false -ckanext.datapusher_plus.qsv_stats_string_max_length = 32767 -ckanext.datapusher_plus.qsv_dates_whitelist = date,time,due,open,close,created -ckanext.datapusher_plus.qsv_freq_limit = 10 -ckanext.datapusher_plus.auto_alias = true -ckanext.datapusher_plus.auto_alias_unique = false -ckanext.datapusher_plus.copy_readbuffer_size = 1048576 -ckanext.datapusher_plus.type_mapping = {"String": "text", "Integer": "numeric","Float": "numeric","DateTime": "timestamp","Date": "date","NULL": "text"} -ckanext.datapusher_plus.auto_spatial_simplication = true -ckanext.datapusher_plus.spatial_simplication_relative_tolerance = 0.1 -ckanext.datapusher_plus.latitude_fields = latitude,lat -ckanext.datapusher_plus.longitude_fields = longitude,long,lon -ckanext.datapusher_plus.jinja2_bytecode_cache_dir = /tmp/jinja2_butecode_cache -ckanext.datapusher_plus.auto_unzip_one_file = true -ckanext.datapusher_plus.api_token = -ckanext.datapusher_plus.describeGPT_api_key = -ckanext.datapusher_plus.file_bin = /usr/bin/file -ckanext.datapusher_plus.enable_druf = false -ckanext.datapusher_plus.enable_form_redirect = true -"#; - std::fs::write("dpp_default_config.ini", dpp_default_config)?; - cmd!( - sh, - "ckan config-tool /etc/ckan/default/ckan.ini -f dpp_default_config.ini" - ) - .run()?; - let resource_formats_str = std::fs::read_to_string( - "/usr/lib/ckan/default/src/ckan/config/resource_formats.json", + step_install_datapusher_plus_extension( + "9.".to_string(), + &sh, + sysadmin_username, + username.clone(), )?; - let mut resource_formats_val: serde_json::Value = - serde_json::from_str(&resource_formats_str)?; - let all_resource_formats = resource_formats_val - .get_mut(0) - .unwrap() - .as_array_mut() - .unwrap(); - all_resource_formats.push(json!([ - "TAB", - "Tab Separated Values File", - "text/tab-separated-values", - [] - ])); - std::fs::write( - "/usr/lib/ckan/default/src/ckan/config/resource_formats.json", - serde_json::to_string(&resource_formats_val)?, - )?; - cmd!(sh, "sudo locale-gen en_US.UTF-8").run()?; - cmd!(sh, "sudo update-locale").run()?; - let token_command_output = cmd!( - sh, - "ckan -c /etc/ckan/default/ckan.ini user token add {sysadmin_username} dpplus" - ) - .read()?; - let tail_output = cmd!(sh, "tail -n 1").stdin(token_command_output).read()?; - let dpp_api_token = cmd!(sh, "tr -d '\t'").stdin(tail_output).read()?; - cmd!(sh, "ckan config-tool /etc/ckan/default/ckan.ini ckanext.datapusher_plus.api_token={dpp_api_token}").env("LC_ALL", "en_US.UTF-8").run()?; - cmd!( - sh, - "ckan -c /etc/ckan/default/ckan.ini db upgrade -p datapusher_plus" - ) - .run()?; - println!( - "{}", - success_text("8. Installed ckanext-scheming and DataPusher+ extensions.") - ); } println!("\n{}", success_text("Running CKAN instance...")); diff --git a/src/steps.rs b/src/steps.rs index 1cc0d3c..7e50d42 100644 --- a/src/steps.rs +++ b/src/steps.rs @@ -1,5 +1,6 @@ use crate::styles::{highlighted_text, important_text, step_text, success_text}; use anyhow::Result; +use serde_json::json; use xshell::{Shell, cmd}; pub fn step_intro() { @@ -135,3 +136,233 @@ POSTGRES_PASSWORD=pass"; ); Ok(()) } + +pub fn step_install_datastore_extension( + step_prefix: String, + sh: &Shell, + username: String, +) -> Result<()> { + println!( + "\n{} Enabling DataStore plugin, adding config URLs in /etc/ckan/default/ckan.ini and updating permissions...", + step_text(step_prefix.as_str()), + ); + let mut conf = ini::Ini::load_from_file("/etc/ckan/default/ckan.ini")?; + let app_main_section = conf.section_mut(Some("app:main")).unwrap(); + let mut ckan_plugins = app_main_section.get("ckan.plugins").unwrap().to_string(); + ckan_plugins.push_str(" datastore"); + app_main_section.insert("ckan.plugins", ckan_plugins); + app_main_section.insert( + "ckan.datastore.write_url", + "postgresql://ckan_default:pass@localhost/datastore_default", + ); + app_main_section.insert( + "ckan.datastore.read_url", + "postgresql://datastore_default:pass@localhost/datastore_default", + ); + app_main_section.insert("ckan.datastore.sqlsearch.enabled", "true"); + conf.write_to_file("/etc/ckan/default/ckan.ini")?; + let postgres_container_id = cmd!( + sh, + "sudo docker ps -aqf name=^ckan-devstaller-project-postgres$" + ) + .read()?; + let set_permissions_output = cmd!( + sh, + "ckan -c /etc/ckan/default/ckan.ini datastore set-permissions" + ) + .read()?; + std::fs::write("permissions.sql", set_permissions_output)?; + loop { + std::thread::sleep(std::time::Duration::from_secs(2)); + if std::fs::exists("permissions.sql")? { + break; + } + } + sh.change_dir(format!("/home/{username}")); + cmd!( + sh, + "sudo docker cp permissions.sql {postgres_container_id}:/permissions.sql" + ) + .run()?; + cmd!(sh, "sudo docker exec {postgres_container_id} psql -U ckan_default --set ON_ERROR_STOP=1 -f permissions.sql").run()?; + println!( + "{}", + success_text( + format!("{step_prefix} Enabled DataStore plugin, set DataStore URLs in /etc/ckan/default/ckan.ini, and updated permissions.").as_str() + ) + ); + Ok(()) +} + +pub fn step_install_ckanext_scheming_extension( + step_prefix: String, + sh: &Shell, + username: String, +) -> Result<()> { + println!( + "{}", + step_text("\n{} Installing the ckanext-scheming extension..."), + ); + cmd!( + sh, + "pip install -e git+https://github.com/ckan/ckanext-scheming.git#egg=ckanext-scheming" + ) + .run()?; + let mut conf = ini::Ini::load_from_file("/etc/ckan/default/ckan.ini")?; + let app_main_section = conf.section_mut(Some("app:main")).unwrap(); + let mut ckan_plugins = app_main_section.get("ckan.plugins").unwrap().to_string(); + ckan_plugins.push_str(" scheming_datasets"); + cmd!( + sh, + "ckan config-tool /etc/ckan/default/ckan.ini -s app:main ckan.plugins={ckan_plugins}" + ) + .run()?; + cmd!(sh, "ckan config-tool /etc/ckan/default/ckan.ini -s app:main scheming.presets=ckanext.scheming:presets.json").run()?; + cmd!( + sh, + "ckan config-tool /etc/ckan/default/ckan.ini -s app:main scheming.dataset_fallback=false" + ) + .run()?; + // app_main_section.insert("ckan.plugins", ckan_plugins); + // app_main_section.insert("scheming.presets", "ckanext.scheming:presets.json"); + // app_main_section.insert("scheming.dataset_fallback", "false"); + // conf.write_to_file("/etc/ckan/default/ckan.ini")?; + Ok(()) +} + +pub fn step_install_datapusher_plus_extension( + step_prefix: String, + sh: &Shell, + sysadmin_username: String, + username: String, +) -> Result<()> { + // Install DataPusher+ + println!( + "{}", + step_text(format!("\n{step_prefix} Installing DataPusher+ extension...").as_str()) + ); + cmd!(sh, "sudo apt install python3-virtualenv python3-dev python3-pip python3-wheel build-essential libxslt1-dev libxml2-dev zlib1g-dev git libffi-dev libpq-dev uchardet -y").run()?; + sh.change_dir("/usr/lib/ckan/default/src"); + cmd!( + sh, + "pip install -e git+https://github.com/dathere/datapusher-plus.git@main#egg=datapusher-plus" + ) + .run()?; + sh.change_dir("/usr/lib/ckan/default/src/datapusher-plus"); + cmd!(sh, "pip install -r requirements.txt").run()?; + sh.change_dir(format!("/home/{username}")); + cmd!(sh, "wget https://github.com/dathere/qsv/releases/download/4.0.0/qsv-4.0.0-x86_64-unknown-linux-gnu.zip").run()?; + cmd!(sh, "sudo apt install unzip -y").run()?; + cmd!(sh, "unzip qsv-4.0.0-x86_64-unknown-linux-gnu.zip").run()?; + cmd!(sh, "sudo rm -rf qsv-4.0.0-x86_64-unknown-linux-gnu.zip").run()?; + cmd!(sh, "sudo mv ./qsvdp_glibc-2.31 /usr/local/bin/qsvdp").run()?; + let mut conf = ini::Ini::load_from_file("/etc/ckan/default/ckan.ini")?; + let app_main_section = conf.section_mut(Some("app:main")).unwrap(); + let mut ckan_plugins = app_main_section.get("ckan.plugins").unwrap().to_string(); + ckan_plugins.push_str(" datapusher_plus"); + cmd!( + sh, + "ckan config-tool /etc/ckan/default/ckan.ini -s app:main ckan.plugins={ckan_plugins}" + ) + .run()?; + cmd!(sh, "ckan config-tool /etc/ckan/default/ckan.ini -s app:main scheming.dataset_schemas=ckanext.datapusher_plus:dataset-druf.yaml").run()?; + // app_main_section.insert("ckan.plugins", ckan_plugins); + // app_main_section.insert( + // "scheming.dataset_schemas", + // "ckanext.datapusher_plus:dataset-druf.yaml", + // ); + // conf.write_to_file("/etc/ckan/default/ckan.ini")?; + let dpp_default_config = r#" +ckanext.datapusher_plus.use_proxy = false +ckanext.datapusher_plus.download_proxy = +ckanext.datapusher_plus.ssl_verify = false +# supports INFO, DEBUG, TRACE - use DEBUG or TRACE when debugging scheming Formulas +ckanext.datapusher_plus.upload_log_level = INFO +ckanext.datapusher_plus.formats = csv tsv tab ssv xls xlsx xlsxb xlsm ods geojson shp qgis zip +ckanext.datapusher_plus.pii_screening = false +ckanext.datapusher_plus.pii_found_abort = false +ckanext.datapusher_plus.pii_regex_resource_id_or_alias = +ckanext.datapusher_plus.pii_show_candidates = false +ckanext.datapusher_plus.pii_quick_screen = false +ckanext.datapusher_plus.qsv_bin = /usr/local/bin/qsvdp +ckanext.datapusher_plus.preview_rows = 100 +ckanext.datapusher_plus.download_timeout = 300 +ckanext.datapusher_plus.max_content_length = 1256000000000 +ckanext.datapusher_plus.chunk_size = 16384 +ckanext.datapusher_plus.default_excel_sheet = 0 +ckanext.datapusher_plus.sort_and_dupe_check = true +ckanext.datapusher_plus.dedup = false +ckanext.datapusher_plus.unsafe_prefix = unsafe_ +ckanext.datapusher_plus.reserved_colnames = _id +ckanext.datapusher_plus.prefer_dmy = false +ckanext.datapusher_plus.ignore_file_hash = true +ckanext.datapusher_plus.auto_index_threshold = 3 +ckanext.datapusher_plus.auto_index_dates = true +ckanext.datapusher_plus.auto_unique_index = true +ckanext.datapusher_plus.summary_stats_options = +ckanext.datapusher_plus.add_summary_stats_resource = false +ckanext.datapusher_plus.summary_stats_with_preview = false +ckanext.datapusher_plus.qsv_stats_string_max_length = 32767 +ckanext.datapusher_plus.qsv_dates_whitelist = date,time,due,open,close,created +ckanext.datapusher_plus.qsv_freq_limit = 10 +ckanext.datapusher_plus.auto_alias = true +ckanext.datapusher_plus.auto_alias_unique = false +ckanext.datapusher_plus.copy_readbuffer_size = 1048576 +ckanext.datapusher_plus.type_mapping = {"String": "text", "Integer": "numeric","Float": "numeric","DateTime": "timestamp","Date": "date","NULL": "text"} +ckanext.datapusher_plus.auto_spatial_simplication = true +ckanext.datapusher_plus.spatial_simplication_relative_tolerance = 0.1 +ckanext.datapusher_plus.latitude_fields = latitude,lat +ckanext.datapusher_plus.longitude_fields = longitude,long,lon +ckanext.datapusher_plus.jinja2_bytecode_cache_dir = /tmp/jinja2_butecode_cache +ckanext.datapusher_plus.auto_unzip_one_file = true +ckanext.datapusher_plus.api_token = +ckanext.datapusher_plus.describeGPT_api_key = +ckanext.datapusher_plus.file_bin = /usr/bin/file +ckanext.datapusher_plus.enable_druf = false +ckanext.datapusher_plus.enable_form_redirect = true +"#; + std::fs::write("dpp_default_config.ini", dpp_default_config)?; + cmd!( + sh, + "ckan config-tool /etc/ckan/default/ckan.ini -f dpp_default_config.ini" + ) + .run()?; + let resource_formats_str = + std::fs::read_to_string("/usr/lib/ckan/default/src/ckan/config/resource_formats.json")?; + let mut resource_formats_val: serde_json::Value = serde_json::from_str(&resource_formats_str)?; + let all_resource_formats = resource_formats_val + .get_mut(0) + .unwrap() + .as_array_mut() + .unwrap(); + all_resource_formats.push(json!([ + "TAB", + "Tab Separated Values File", + "text/tab-separated-values", + [] + ])); + std::fs::write( + "/usr/lib/ckan/default/src/ckan/config/resource_formats.json", + serde_json::to_string(&resource_formats_val)?, + )?; + cmd!(sh, "sudo locale-gen en_US.UTF-8").run()?; + cmd!(sh, "sudo update-locale").run()?; + let token_command_output = cmd!( + sh, + "ckan -c /etc/ckan/default/ckan.ini user token add {sysadmin_username} dpplus" + ) + .read()?; + let tail_output = cmd!(sh, "tail -n 1").stdin(token_command_output).read()?; + let dpp_api_token = cmd!(sh, "tr -d '\t'").stdin(tail_output).read()?; + cmd!(sh, "ckan config-tool /etc/ckan/default/ckan.ini ckanext.datapusher_plus.api_token={dpp_api_token}").env("LC_ALL", "en_US.UTF-8").run()?; + cmd!( + sh, + "ckan -c /etc/ckan/default/ckan.ini db upgrade -p datapusher_plus" + ) + .run()?; + println!( + "{}", + success_text(format!("{step_prefix} Installed DataPusher+ extension.").as_str()) + ); + Ok(()) +}