diff --git a/.github/workflows/run-ckan-only-install.yml b/.github/workflows/run-ckan-only-install.yml new file mode 100644 index 0000000..68ea266 --- /dev/null +++ b/.github/workflows/run-ckan-only-install.yml @@ -0,0 +1,26 @@ +name: Verify ckan-devstaller runs CKAN-only install successfully +on: + push: + branches: + - main + paths-ignore: + - 'docs/**' + - 'README.md' + workflow_dispatch: +jobs: + ckanonlyinstall: + name: Run ckan-devstaller with a CKAN-only install + runs-on: ubuntu-22.04 + permissions: + # For the git-auto-commit-action + contents: write + steps: + - uses: actions/checkout@v4 + - name: Install Rust stable toolchain + uses: dtolnay/rust-toolchain@stable + - name: Cache cargo deps and target folder + uses: Swatinem/rust-cache@v2 + with: + workspaces: ". -> target" + - name: Run ckan-devstaller + run: cargo run --release --verbose -- --ckan-version 2.11.4 --skip-interactive --skip-run diff --git a/README.md b/README.md index cf61f7a..762f607 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ {1329F0BA-A29F-4BF8-BB6B-E3BA84FDAFCC} -`ckan-devstaller` attempts to install a CKAN instance using [ckan-compose](https://github.com/tino097/ckan-compose) for development usage in a new Ubuntu 22.04 instance. +`ckan-devstaller` attempts to install a [CKAN](https://ckan.org) instance using [ckan-compose](https://github.com/tino097/ckan-compose/tree/ckan-devstaller) for development usage in a new Ubuntu 22.04 instance. You may find `ckan-devstaller` useful for: @@ -29,6 +29,7 @@ You may find the following guides useful while developing with CKAN: - [Customize your config file](https://docs.ckan.org/en/latest/extensions/index.html) - [Create test data](https://docs.ckan.org/en/latest/maintaining/getting-started.html#creating-test-data) - [Visit ckan.org](https://ckan.org) +- [Interact with your CKAN API at ckanaction.dathere.com](https://ckanaction.dathere.com) ## `ckan-devstaller` demos diff --git a/docs/app/(home)/page.tsx b/docs/app/(home)/page.tsx index 63885c7..72a91c6 100644 --- a/docs/app/(home)/page.tsx +++ b/docs/app/(home)/page.tsx @@ -1,6 +1,7 @@ /** biome-ignore-all lint/suspicious/noArrayIndexKey: Would need to look into this trivial issue */ "use client"; +import { CodeBlock } from "fumadocs-ui/components/codeblock"; import defaultMdxComponents from "fumadocs-ui/mdx"; import { cn } from "fumadocs-ui/utils/cn"; import { @@ -8,11 +9,14 @@ import { GitMergeIcon, HomeIcon, SailboatIcon, + TerminalIcon, + Trash2Icon, ZapIcon, } from "lucide-react"; import Image from "next/image"; import Link from "next/link"; -import { useState } from "react"; +import { type HTMLProps, type ReactNode, useState } from "react"; +import { Pre } from "@/components/codeblock"; import { buttonVariants } from "@/components/ui/button"; import CkanDevstallerDemo from "./ckan-devstaller-demo.gif"; @@ -37,8 +41,34 @@ export default function HomePage() { >
+
+
+ ); @@ -88,7 +118,7 @@ function Hero() {

- - } - href="/docs/quick-start" - title="Quick start" - > - Get started with ckan-devstaller and install CKAN within minutes - - } href="/docs/builder" title="Builder"> - Customize your installation with an interactive web GUI - - } - href="/docs/reference/installation-architecture" - title="Installation architecture" - > - Learn about where files are installed after running - ckan-devstaller - - } - href="https://github.com/dathere/ckan-devstaller" - title="Source code" - > - View the source code of ckan-devstaller on GitHub - - + + } href="/docs/builder" title="Quick start"> + Get started with ckan-devstaller and install CKAN within minutes + + } href="/docs/builder" title="Builder"> + Customize your installation with an interactive web GUI + + } + href="/docs/reference/installation-architecture" + title="Installation architecture" + > + Learn about where files are installed after running ckan-devstaller + + } + href="https://github.com/dathere/ckan-devstaller" + title="Source code" + > + View the source code of ckan-devstaller on GitHub + + ); @@ -150,7 +175,7 @@ function PreviewImages() { ]; return ( -
+
{/*
@@ -184,3 +209,207 @@ function PreviewImages() {
); } + +function Why() { + return ( +
+ +
./ckan-devstaller
+ + } + codeblockUninstall={ + +
./ckan-devstaller uninstall
+
+ } + /> +
+ ); +} + +function WhyInteractive(props: { + codeblockInstall: ReactNode; + codeblockUninstall: ReactNode; +}) { + const [active, setActive] = useState(0); + const items = [ + [ + , + "Install CKAN within minutes", + ], + [ + , + "Customize your installation", + ], + [ + , + "Designed for developers", + ], + [ + , + "Uninstall with ease", + ], + ]; + + return ( +
+
+ {items.map((item, i) => ( + + ))} +
+ + +
+ {active === 0 ? ( + +

+ + Install CKAN within minutes. +

+

+ One of the primary goals of ckan-devstaller is to ease + installation of CKAN for development. Built with Rust for speed + and streamlining installation with{" "} + + ckan-compose + + , ckan-devstaller improves installation speeds{" "} + from hours/days to just minutes depending on your + download speed. +

+
+ + Get started + +
+
+ ) : null} + {active === 1 ? ( + +

+ + Customize your installation with the Builder. +

+

+ Try out the interactive web GUI for customizing your CKAN + installation. You can select: +

+
    +
  • Presets
  • +
  • CKAN version
  • +
  • Extensions
  • +
  • Features
  • +
+

+ Then you can copy the provided ckan-devstaller command to run your + selected configuration. +

+
+ + Try out the Builder + +
+
+ ) : null} + {active === 2 ? ( + +

+ + Designed for developers. +

+

+ We've kept development use cases in mind while developing + ckan-devstaller, such as: +

+
    +
  • Trying out a new version of CKAN
  • +
  • Developing CKAN extensions and themes
  • +
+
+ + View the installation architecture + + + Source code + +
+
+ ) : null} + {active === 3 ? ( + +

+ + Uninstall CKAN with ease. +

+

+ After you've installed CKAN with ckan-devstaller, you can + uninstall CKAN with ease. This allows for quickly re-installing + CKAN for a different use case. +

+ {props.codeblockUninstall} + + Learn more about uninstalling + +
+ ) : null} +
+
+ ); +} + +function WhyPanel(props: HTMLProps) { + return ( +
+ {props.children} +
+ ); +} diff --git a/docs/app/global.css b/docs/app/global.css index f415641..b6836ec 100644 --- a/docs/app/global.css +++ b/docs/app/global.css @@ -2,4 +2,4 @@ @import "fumadocs-ui/css/neutral.css"; @import "fumadocs-ui/css/ocean.css"; @import "fumadocs-ui/css/preset.css"; -@import "tw-animate-css"; +@import "tw-animate-css"; \ No newline at end of file diff --git a/docs/bun.lock b/docs/bun.lock index 3c1d10d..d5096fe 100644 --- a/docs/bun.lock +++ b/docs/bun.lock @@ -6,6 +6,8 @@ "dependencies": { "@radix-ui/react-accordion": "^1.2.12", "@radix-ui/react-collapsible": "^1.1.12", + "@radix-ui/react-label": "^2.1.7", + "@radix-ui/react-switch": "^1.2.6", "@radix-ui/react-tabs": "^1.1.13", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", @@ -227,6 +229,8 @@ "@radix-ui/react-id": ["@radix-ui/react-id@1.1.1", "", { "dependencies": { "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-kGkGegYIdQsOb4XjsfM97rXsiHaBwco+hFI66oO4s9LU+PLAC5oJ7khdOVFxkhsmlbpUqDAvXw11CluXP+jkHg=="], + "@radix-ui/react-label": ["@radix-ui/react-label@2.1.7", "", { "dependencies": { "@radix-ui/react-primitive": "2.1.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-YT1GqPSL8kJn20djelMX7/cTRp/Y9w5IZHvfxQTVHrOqa2yMl7i/UfMqKRU5V7mEyKTrUVgJXhNQPVCG8PBLoQ=="], + "@radix-ui/react-navigation-menu": ["@radix-ui/react-navigation-menu@1.2.14", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-collection": "1.1.7", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-dismissable-layer": "1.1.11", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-presence": "1.1.5", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-callback-ref": "1.1.1", "@radix-ui/react-use-controllable-state": "1.2.2", "@radix-ui/react-use-layout-effect": "1.1.1", "@radix-ui/react-use-previous": "1.1.1", "@radix-ui/react-visually-hidden": "1.2.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-YB9mTFQvCOAQMHU+C/jVl96WmuWeltyUEpRJJky51huhds5W2FQr1J8D/16sQlf0ozxkPK8uF3niQMdUwZPv5w=="], "@radix-ui/react-popover": ["@radix-ui/react-popover@1.1.15", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-dismissable-layer": "1.1.11", "@radix-ui/react-focus-guards": "1.1.3", "@radix-ui/react-focus-scope": "1.1.7", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-popper": "1.2.8", "@radix-ui/react-portal": "1.1.9", "@radix-ui/react-presence": "1.1.5", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-slot": "1.2.3", "@radix-ui/react-use-controllable-state": "1.2.2", "aria-hidden": "^1.2.4", "react-remove-scroll": "^2.6.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-kr0X2+6Yy/vJzLYJUPCZEc8SfQcf+1COFoAqauJm74umQhta9M7lNJHP7QQS3vkvcGLQUbWpMzwrXYwrYztHKA=="], @@ -245,6 +249,8 @@ "@radix-ui/react-slot": ["@radix-ui/react-slot@1.2.3", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-aeNmHnBxbi2St0au6VBVC7JXFlhLlOnvIIlePNniyUNAClzmtAUEY8/pBiK3iHjufOlwA+c20/8jngo7xcrg8A=="], + "@radix-ui/react-switch": ["@radix-ui/react-switch@1.2.6", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-controllable-state": "1.2.2", "@radix-ui/react-use-previous": "1.1.1", "@radix-ui/react-use-size": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-bByzr1+ep1zk4VubeEVViV592vu2lHE2BZY5OnzehZqOOgogN80+mNtCqPkhn2gklJqOpxWgPoYTSnhBCqpOXQ=="], + "@radix-ui/react-tabs": ["@radix-ui/react-tabs@1.1.13", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-presence": "1.1.5", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-roving-focus": "1.1.11", "@radix-ui/react-use-controllable-state": "1.2.2" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-7xdcatg7/U+7+Udyoj2zodtI9H/IIopqo+YOIcZOq1nJwXWBZ9p8xiu5llXlekDbZkca79a/fozEYQXIA4sW6A=="], "@radix-ui/react-use-callback-ref": ["@radix-ui/react-use-callback-ref@1.1.1", "", { "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-FkBMwD+qbGQeMu1cOHnuGB6x4yzPjho8ap5WtbEJ26umhgqVXbhekKUQO+hZEL1vU92a3wHwdp0HAcqAUF5iDg=="], diff --git a/docs/components/builder-sections/ckan-version.tsx b/docs/components/builder-sections/ckan-version.tsx index 2348d8b..d286068 100644 --- a/docs/components/builder-sections/ckan-version.tsx +++ b/docs/components/builder-sections/ckan-version.tsx @@ -11,26 +11,26 @@ export default function CKANVersionBuilderSection({ config, setConfig }: any) { } - title="2.11.3" + title="2.11.4" className={ - config.ckanVersion === "2.11.3" + config.ckanVersion === "2.11.4" ? selectedCardClasses : "cursor-pointer" } onClick={() => { - setConfig({ ...config, ckanVersion: "2.11.3" }); + setConfig({ ...config, ckanVersion: "2.11.4" }); }} > } - title="2.10.8" + title="2.10.9" className={ - config.ckanVersion === "2.10.8" + config.ckanVersion === "2.10.9" ? selectedCardClasses : "cursor-pointer" } onClick={() => { - setConfig({ ...config, ckanVersion: "2.10.8" }); + setConfig({ ...config, ckanVersion: "2.10.9" }); }} > diff --git a/docs/components/builder-sections/presets.tsx b/docs/components/builder-sections/presets.tsx index 444652b..4f0c276 100644 --- a/docs/components/builder-sections/presets.tsx +++ b/docs/components/builder-sections/presets.tsx @@ -49,7 +49,7 @@ export default function PresetsBuilderSection({ setConfig({ ...config, preset: "dathere-default", - ckanVersion: "2.11.3", + ckanVersion: "2.11.4", extensions: ["ckanext-scheming", "DataStore", "DataPusher+"], features: ["enable-ssh"], }); diff --git a/docs/components/builder.tsx b/docs/components/builder.tsx index d5402da..6b96529 100644 --- a/docs/components/builder.tsx +++ b/docs/components/builder.tsx @@ -1,18 +1,13 @@ "use client"; import { CodeBlock, Pre } from "fumadocs-ui/components/codeblock"; -import defaultMdxComponents from "fumadocs-ui/mdx"; -import { - BarChartBigIcon, - BlocksIcon, - SailboatIcon, - TerminalSquareIcon, -} from "lucide-react"; +import { Label } from "@/components/ui/label" +import { Switch } from "@/components/ui/switch" import { useEffect, useState } from "react"; -import PresetsBuilderSection from "./builder-sections/presets"; -import CKANVersionBuilderSection from "./builder-sections/ckan-version"; -import CKANExtensionsBuilderSection from "./builder-sections/ckan-extensions"; -import FeaturesBuilderSection from "./builder-sections/features"; +import PresetsBuilderSection from "@/components/builder-sections/presets"; +import CKANVersionBuilderSection from "@/components/builder-sections/ckan-version"; +import CKANExtensionsBuilderSection from "@/components/builder-sections/ckan-extensions"; +import FeaturesBuilderSection from "@/components/builder-sections/features"; export type Config = { preset: string | undefined; @@ -25,11 +20,28 @@ export const selectedCardClasses = "bg-blue-100 dark:bg-blue-950 border-blue-300 dark:border-blue-900 border-2"; export default function Builder() { - const { Card, Cards } = defaultMdxComponents; + const [downloadScript, setDownloadScript] = useState(true); + const downloadScriptString = `#!/usr/bin/env bash + +# Update/Upgrade system dependencies +sudo apt update -y +sudo apt upgrade -y + +# Install curl +sudo apt install curl -y + +# Change to the home directory +cd ~/ + +# Download the ckan-devstaller binary file +wget https://github.com/dathere/ckan-devstaller/releases/download/0.3.1/ckan-devstaller + +# Add execute permission to ckan-devstaller binary file +sudo chmod +x ./ckan-devstaller\n\n# Run ckan-devstaller script\n`; const [command, setCommand] = useState("./ckan-devstaller"); const [config, setConfig] = useState({ preset: "ckan-only", - ckanVersion: "2.11.3", + ckanVersion: "2.11.4", extensions: [], features: [], }); @@ -52,12 +64,16 @@ ${ckanVersionString}${extensionsString ? extensionsString : ""}${featuresString }, [config]); return ( -
-
+
+

ckan-devstaller command

+
+ setDownloadScript(!downloadScript)} id="download-script" /> + +
-
{command}
+
{downloadScript ? downloadScriptString : ""}{command}

Selected configuration

@@ -87,7 +103,7 @@ ${ckanVersionString}${extensionsString ? extensionsString : ""}${featuresString
-
+

Configuration options

diff --git a/docs/components/steps.tsx b/docs/components/steps.tsx new file mode 100644 index 0000000..ce348a2 --- /dev/null +++ b/docs/components/steps.tsx @@ -0,0 +1,9 @@ +import type { ReactNode } from 'react'; + +export function Steps({ children }: { children: ReactNode }) { + return
{children}
; +} + +export function Step({ children }: { children: ReactNode }) { + return
{children}
; +} diff --git a/docs/components/ui/button.tsx b/docs/components/ui/button.tsx index 9fb2a80..c614777 100644 --- a/docs/components/ui/button.tsx +++ b/docs/components/ui/button.tsx @@ -20,6 +20,7 @@ const buttonVariants = cva( sm: "h-9 px-3", lg: "h-11 px-6", xs: "px-2 py-1.5 text-xs", + "icon-xs": "p-1 [&_svg]:size-4" }, }, defaultVariants: { diff --git a/docs/components/ui/label.tsx b/docs/components/ui/label.tsx new file mode 100644 index 0000000..5341821 --- /dev/null +++ b/docs/components/ui/label.tsx @@ -0,0 +1,26 @@ +"use client" + +import * as React from "react" +import * as LabelPrimitive from "@radix-ui/react-label" +import { cva, type VariantProps } from "class-variance-authority" + +import { cn } from "@/lib/utils" + +const labelVariants = cva( + "text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70" +) + +const Label = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef & + VariantProps +>(({ className, ...props }, ref) => ( + +)) +Label.displayName = LabelPrimitive.Root.displayName + +export { Label } diff --git a/docs/components/ui/switch.tsx b/docs/components/ui/switch.tsx new file mode 100644 index 0000000..9511e19 --- /dev/null +++ b/docs/components/ui/switch.tsx @@ -0,0 +1,29 @@ +"use client" + +import * as React from "react" +import * as SwitchPrimitives from "@radix-ui/react-switch" + +import { cn } from "@/lib/utils" + +const Switch = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + + + +)) +Switch.displayName = SwitchPrimitives.Root.displayName + +export { Switch } diff --git a/docs/content/docs/builder.mdx b/docs/content/docs/builder.mdx index 565a7b9..41701ea 100644 --- a/docs/content/docs/builder.mdx +++ b/docs/content/docs/builder.mdx @@ -4,6 +4,19 @@ description: Customize your CKAN installation before running ckan-devstaller. icon: Blocks --- +ckan-devstaller attempts to install a CKAN instance from source along with [ckan-compose](https://github.com/tino097/ckan-compose/tree/ckan-devstaller) and other optional features, intended for development use in a new Ubuntu 22.04 instance. + +Make sure `ckan-devstaller` is run in a **new** Ubuntu 22.04 instance. Do NOT run `ckan-devstaller` in an existing instance that is important for your usage. + +import { Accordion, Accordions } from 'fumadocs-ui/components/accordion'; + + + If you are using Ubuntu 22.04 on VirtualBox, you may need to add your user to the sudoers file before running the ckan-devstaller install script. Open a terminal in your virtual machine (VM), run `su -` and log in as the root user with the password you used to set up the VM, then type:

`sudo adduser sudo`

where `` is your username then restart your VM and run the ckan-devstaller installer script.
+ Currently `ckan-devstaller` supports `x86_64` architecture. `ARM64` support is planned. +
+ +--- + import Builder from "@/components/builder"; diff --git a/docs/content/docs/changelog/0.3.0.mdx b/docs/content/docs/changelog/0.3.0.mdx new file mode 100644 index 0000000..b8143a7 --- /dev/null +++ b/docs/content/docs/changelog/0.3.0.mdx @@ -0,0 +1,37 @@ +--- +title: Changelog for ckan-devstaller v0.3.0 (2025-10-14) +--- + +Since v0.2.1 of ckan-devstaller, there have been many new features and changes now available in v0.3.0. + +## New web app: ckan-devstaller.dathere.com + +We've released a new web app [ckan-devstaller.dathere.com](https://ckan-devstaller.dathere.com) as the primary documentation site for ckan-devstaller. + +## Builder page + +There is now an interactive web GUI, the [Builder](/docs/builder), for users to customize their CKAN installation before copying the (now updated) ckan-devstaller command and running it on their terminal. This helps resolve issue [#6](https://github.com/dathere/ckan-devstaller/issues/11). + +## Updated Quick Start page + +The [Quick Start](/docs) page now includes three options for suggested installation methods: + +1. Use the interactive [Builder](/docs/builder). +2. Install the "CKAN-only" preset with a script which installs the latest stable version of CKAN and ckan-compose (with optional non-interactive script). +3. Install the "datHere default" preset with a script which installs the latest stable version of CKAN and ckan-compose (with optional non-interactive script) along with the DataStore, ckanext-scheming, and DataPusher+ extensions and also installs the `openssh-server` package. + +## Installation architecture page + +There is now an [Installation Architecture](/docs/reference/installation-architecture) page in the Reference section of the web app that provides a visual representation of where `ckan-devstaller` installs relevant files/folders. + +## Uninstall CKAN page + +There is now an [Uninstall CKAN](/docs/tutorials/uninstall-ckan) page in the Tutorials section of the web app that helps users understand how to uninstall their newly installed CKAN installation. This includes the option to either use the new `ckan-devstaller uninstall` subcommand or run the script directly. + +## README update + +The README on the [ckan-devstaller GitHub repository](https://github.com/dathere/ckan-devstaller) has been updated to have a more user-friendly focus for users and developers that may be new to CKAN thanks to the suggestions by [@drw](https://github.com/drw) in issues [#10](https://github.com/dathere/ckan-devstaller/issues/10) and [#11](https://github.com/dathere/ckan-devstaller/issues/11). + +## Changelog section + +We've added a Changelog section to the web app to denote new changes to `ckan-devstaller` for each release. diff --git a/docs/content/docs/changelog/0.3.1.mdx b/docs/content/docs/changelog/0.3.1.mdx new file mode 100644 index 0000000..536bde0 --- /dev/null +++ b/docs/content/docs/changelog/0.3.1.mdx @@ -0,0 +1,37 @@ +--- +title: Changelog for ckan-devstaller v0.3.1 (2025-10-30) +--- + +## Updated CKAN default versions to latest stable versions + +We have updated ckan-devstaller to suggest the recent releases of CKAN [2.11.4](https://docs.ckan.org/en/2.11/changelog.html#v-2-11-4-2025-10-29) and [2.10.9](https://docs.ckan.org/en/2.10/changelog.html#v-2-10-9-2025-10-29). + +## Added the "Developing with WSL" page in the Reference section + +Developers using Windows may benefit from the new [Developing with WSL](/docs/reference/developing-with-wsl) page in the Reference section to try their builds of ckan-devstaller and verify things work on a new install of Ubuntu 22.04. + +Alternatively if not trying to reset an entire WSL environment, developers can look into [`ckan-devstaller uninstall`](/docs/tutorials/uninstall-ckan). + +## Added install script switch to Builder page + +There is now a switch on the Builder page that is enabled by default for including the installation script for ckan-devstaller before running it. This is necessary for users that don't have ckan-devstaller installed yet and are using the Builder over the Quick Start scripts. + +## Default start with the Builder page + +We've changed "Get Started" links (and similar links) to the Builder page instead of the Quick Start page to users can quickly access the Builder again and get started using ckan-devstaller. + +## Update ckan-compose links to use ckan-devstaller branch + +We've updated ckan-compose links to use [the ckan-devstaller branch](https://github.com/tino097/ckan-compose/tree/ckan-devstaller). + +## Update VirtualBox notice with better formatting + +To ensure users using VirtualBox see the full command for adding a user to the sudoers file, we've improved the formatting of the note. + +## GitHub Action to verify CKAN install runs + +We've added a GitHub Action to run on push to the main branch to verify that a CKAN-only install runs. This doesn't include verbose testing but rather ensuring that the ckan-devstaller finishes without errors. + +## New --skip-run flag + +We've added a `--skip-run` flag to skip running CKAN at the end of installation. diff --git a/docs/content/docs/index.mdx b/docs/content/docs/index.mdx index d6f550b..ffa0d8d 100644 --- a/docs/content/docs/index.mdx +++ b/docs/content/docs/index.mdx @@ -4,22 +4,29 @@ description: Get started with ckan-devstaller and install CKAN within minutes. icon: Zap --- -ckan-devstaller attempts to install a CKAN instance from source along with [ckan-compose](https://github.com/tino097/ckan-compose) and other optional features, intended for development use in a new Ubuntu 22.04 instance. +ckan-devstaller attempts to install a CKAN instance from source along with [ckan-compose](https://github.com/tino097/ckan-compose/tree/ckan-devstaller) and other optional features, intended for development use in a new Ubuntu 22.04 instance. Make sure `ckan-devstaller` is run in a **new** Ubuntu 22.04 instance. Do NOT run `ckan-devstaller` in an existing instance that is important for your usage. import { Accordion, Accordions } from 'fumadocs-ui/components/accordion'; - If you are using Ubuntu 22.04 on VirtualBox, you may need to add your user to the sudoers file before running the ckan-devstaller install script. Open a terminal in your virtual machine (VM), run `su -` and log in as the root user with the password you used to set up the VM, then type `sudo adduser sudo` where `` is your username then restart your VM and run the ckan-devstaller installer script. + If you are using Ubuntu 22.04 on VirtualBox, you may need to add your user to the sudoers file before running the ckan-devstaller install script. Open a terminal in your virtual machine (VM), run `su -` and log in as the root user with the password you used to set up the VM, then type:

`sudo adduser sudo`

where `` is your username then restart your VM and run the ckan-devstaller installer script.
Currently `ckan-devstaller` supports `x86_64` architecture. `ARM64` support is planned.
--- -You have several options to choose from for installation. Here are a few: +## Install CKAN using ckan-devstaller -## [1/3] Customize your CKAN installation with the Builder (Recommended) +You have several options to choose from for installation. Here are a few you may choose one from: + +import { Step, Steps } from 'fumadocs-ui/components/steps'; + + + + +### Customize your CKAN installation with the Builder (Recommended) } @@ -29,25 +36,33 @@ You have several options to choose from for installation. Here are a few: Click here to customize your CKAN installation with an interactive web GUI -## [2/3] Install the "CKAN-only" preset + -By running the following script, ckan-devstaller will be downloaded and the default configuration for installing CKAN 2.11.3 with ckan-compose will be selected. You can then customize your configuration interactively in your terminal after running this script. + + +### Install the "CKAN-only" preset + +By running the following script, ckan-devstaller will be downloaded and the default configuration for installing CKAN with ckan-compose will be selected. You can then customize your configuration interactively in your terminal after running this script. ```bash -wget -O - https://github.com/dathere/ckan-devstaller/releases/download/0.3.0/install.bash | bash +wget -O - https://github.com/dathere/ckan-devstaller/releases/download/0.3.1/install.bash | bash ``` If you'd rather skip the interactivity and go straight to installation, then run the following script instead: ```bash -wget -O - https://github.com/dathere/ckan-devstaller/releases/download/0.3.0/install.bash | bash -s skip-interactive +wget -O - https://github.com/dathere/ckan-devstaller/releases/download/0.3.1/install.bash | bash -s skip-interactive ``` -## [3/3] Install the "datHere Default" preset + + + + +### Install the "datHere Default" preset The following script will download ckan-devstaller and select the following configuration: -- CKAN 2.11.3 +- CKAN latest stable version - [ckan-compose](https://github.com/tino097/ckan-compose/tree/ckan-devstaller) - [DataStore extension](https://docs.ckan.org/en/2.11/maintaining/datastore.html) - [ckanext-scheming extension](https://github.com/ckan/ckanext-scheming) @@ -58,15 +73,18 @@ The following script will download ckan-devstaller and select the following conf You can then customize your configuration interactively in your terminal after running this script. ```bash -wget -O - https://github.com/dathere/ckan-devstaller/releases/download/0.3.0/install.bash | bash -s dathere-default +wget -O - https://github.com/dathere/ckan-devstaller/releases/download/0.3.1/install.bash | bash -s dathere-default ``` If you'd rather skip the interactivity and go straight to installation, then run the following script instead: ```bash -wget -O - https://github.com/dathere/ckan-devstaller/releases/download/0.3.0/install.bash | bash -s dathere-default skip-interactive +wget -O - https://github.com/dathere/ckan-devstaller/releases/download/0.3.1/install.bash | bash -s dathere-default skip-interactive ``` + + + ## Learn more import { BlocksIcon, HomeIcon, GitMergeIcon, Trash2Icon } from 'lucide-react'; diff --git a/docs/content/docs/meta.json b/docs/content/docs/meta.json index 71c5117..608d5ba 100644 --- a/docs/content/docs/meta.json +++ b/docs/content/docs/meta.json @@ -5,6 +5,9 @@ "builder", "---Further reading---", "tutorials", - "reference" + "reference", + "changelog", + "--- ---", + "[Privacy Policy](https://dathere.com/privacy-policy/)" ] } \ No newline at end of file diff --git a/docs/content/docs/reference/developing-with-wsl.mdx b/docs/content/docs/reference/developing-with-wsl.mdx new file mode 100644 index 0000000..c413785 --- /dev/null +++ b/docs/content/docs/reference/developing-with-wsl.mdx @@ -0,0 +1,131 @@ +--- +title: Developing with WSL +description: Tips on how to develop ckan-devstaller on a Windows machine by leveraging Windows Subsystem for Linux. +--- + +When developing ckan-devstaller on Windows, using Windows Subsystem for Linux (WSL) can be advantageous to demo an Ubuntu 22.04 environment without having to set up a virtual machine. + +import { Step, Steps } from 'fumadocs-ui/components/steps'; +import { File, Folder, Files } from 'fumadocs-ui/components/files'; + + + + +### Install the Ubuntu-22.04 distribution + +You'll need to have the Ubuntu-22.04 distribution installed by running the following command: + +```bash +wsl --install Ubuntu-22.04 --version 2 +``` + + + + + +### Export a base image + + + +Here's the expected set up where we create a WSL folder: + +```files +/c/Users/rzmk/WSL +├── images +| ├── ubuntu-22-04-snapshot.tar +├── instances +| ├── cdr.vdhx +``` + +First we'll generate the `images/ubuntu-22-04-snapshot.tar` file: + +```bash +wsl --export Ubuntu-22.04 ./images/ubuntu-22-04-snapshot.tar +``` + + + +### Generate a VDHX file for our new instance + +Next we'll generate the `instances/cdr.vdhx` file, so we can run the following: + +```bash +wsl --import cdr ./instances ./images/ubuntu-22-04-snapshot.tar +``` + + + + + +### Access your new instance as root + +Now try to access your new Ubuntu 22.04 instance `cdr` as the `root` user by running: + +```bash +wsl -d cdr +``` + + + + + +### Set yourself as the admin user on the instance and log in as the admin user + +Once logged in as `root`, you'll want to edit the `/etc/wsl.conf` file and modify it so you can login as an admin user instead of `root`. We can use the `nano` editor to modify the file: + +```bash +nano /etc/wsl.conf +``` + +Modify the file by adding a `[user]` section with the value `default=rzmk` (where `rzmk` is your username). The file should look similar to this: + +```ini +[boot] +systemd=true + +[user] +default=rzmk +``` + +Now leave your instance: + +```bash +exit +``` + +Then terminate the instance: + +```bash +wsl --terminate cdr +``` + +Run the instance again and you should be logged in as your admin user by default now: + +```bash +wsl -d cdr +``` + + + + + +### Install ckan-devstaller + +Great, now you can go to the home directory and run one of the [quick start](/docs) scripts: + +```bash +cd ~/ +``` + + + + +## Removing your instance + +When you want to remove your instance (e.g. so that you can start a brand new instance) then run the following to unregister it: + +```bash +wsl --unregister cdr +``` + +Then you can follow the steps again from step 3 to generate and try out a new instance. diff --git a/docs/content/docs/reference/installation-architecture.mdx b/docs/content/docs/reference/installation-architecture.mdx index 87c255e..16369f0 100644 --- a/docs/content/docs/reference/installation-architecture.mdx +++ b/docs/content/docs/reference/installation-architecture.mdx @@ -27,7 +27,7 @@ The configuration file for CKAN is installed at `/etc/ckan/default/ckan.ini`: ## ckan-compose -We install certain first-time install files and `ckan-compose` as a directory in the user's home (`~`) directory. For example for the user `adam`: +We install certain first-time install files and [`ckan-compose`](https://github.com/tino097/ckan-compose/tree/ckan-devstaller) as a directory in the user's home (`~`) directory. For example for the user `adam`: ```files /home/adam diff --git a/docs/content/docs/reference/meta.json b/docs/content/docs/reference/meta.json new file mode 100644 index 0000000..0d08c26 --- /dev/null +++ b/docs/content/docs/reference/meta.json @@ -0,0 +1,6 @@ +{ + "pages": [ + "installation-architecture", + "developing-with-wsl" + ] +} \ No newline at end of file diff --git a/docs/package.json b/docs/package.json index a13a8ad..52dd9d0 100644 --- a/docs/package.json +++ b/docs/package.json @@ -13,6 +13,8 @@ "dependencies": { "@radix-ui/react-accordion": "^1.2.12", "@radix-ui/react-collapsible": "^1.1.12", + "@radix-ui/react-label": "^2.1.7", + "@radix-ui/react-switch": "^1.2.6", "@radix-ui/react-tabs": "^1.1.13", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", diff --git a/install.bash b/install.bash index dc0ec57..b8b0001 100644 --- a/install.bash +++ b/install.bash @@ -11,7 +11,7 @@ sudo apt install curl -y cd ~/ # Download the ckan-devstaller binary file -curl -LO https://github.com/dathere/ckan-devstaller/releases/download/0.3.0/ckan-devstaller +curl -LO https://github.com/dathere/ckan-devstaller/releases/download/0.3.1/ckan-devstaller # Add execute permission to ckan-devstaller binary file sudo chmod +x ./ckan-devstaller @@ -22,9 +22,9 @@ skip_interactive=$2 if [ $preset == "dathere-default" ]; then if [ $skip_interactive == "skip-interactive" ]; then - ./ckan-devstaller --ckan-version 2.11.3 --extensions ckanext-scheming DataStore DataPusher+ --features enable-ssh --skip-interactive + ./ckan-devstaller --ckan-version 2.11.4 --extensions ckanext-scheming DataStore DataPusher+ --features enable-ssh --skip-interactive else - ./ckan-devstaller --ckan-version 2.11.3 --extensions ckanext-scheming DataStore DataPusher+ --features enable-ssh + ./ckan-devstaller --ckan-version 2.11.4 --extensions ckanext-scheming DataStore DataPusher+ --features enable-ssh fi else if [ $preset == "skip-interactive" ]; then diff --git a/src/main.rs b/src/main.rs index 8931861..f304792 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,7 +5,9 @@ mod styles; use crate::{ questions::{question_ckan_version, question_ssh, question_sysadmin}, steps::{ - step_install_ahoy, step_install_and_run_ckan_compose, step_install_curl, + step_install_ahoy, step_install_and_run_ckan_compose, + step_install_ckanext_scheming_extension, step_install_curl, + step_install_datapusher_plus_extension, step_install_datastore_extension, step_install_docker, step_install_openssh, step_package_updates, }, styles::{important_text, step_text, success_text}, @@ -14,7 +16,6 @@ use anyhow::Result; use clap::{Parser, Subcommand}; use human_panic::{metadata, setup_panic}; use inquire::Confirm; -use serde_json::json; use std::{path::PathBuf, str::FromStr}; use xshell::cmd; use xshell_venv::{Shell, VirtualEnv}; @@ -26,6 +27,9 @@ struct Args { /// Skip interactive steps #[arg(short, long)] skip_interactive: bool, + /// Skip running CKAN at the end of installation + #[arg(short, long)] + skip_run: bool, #[arg(short, long)] /// CKAN version to install defined by semantic versioning from official releases from https://github.com/ckan/ckan ckan_version: Option, @@ -109,7 +113,7 @@ rm -rf README ckan-compose ahoy dpp_default_config.ini get-docker.sh permissions ckan_version: if args.ckan_version.is_some() { args.ckan_version.unwrap() } else { - "2.11.3".to_string() + "2.11.4".to_string() }, sysadmin: default_sysadmin.clone(), extension_datastore: args @@ -133,7 +137,7 @@ rm -rf README ckan-compose ahoy dpp_default_config.ini get-docker.sh permissions if config.ssh { default_config_text.push_str("\n- Install openssh-server to enable SSH access"); } - default_config_text.push_str("\n- Install ckan-compose (https://github.com/tino097/ckan-compose) which sets up the CKAN backend (PostgreSQL, SOLR, Redis)"); + default_config_text.push_str("\n- Install ckan-compose (https://github.com/tino097/ckan-compose/tree/ckan-devstaller) which sets up the CKAN backend (PostgreSQL, SOLR, Redis)"); default_config_text.push_str(format!("\n- Install CKAN v{}", config.ckan_version).as_str()); if config.extension_datastore { default_config_text.push_str("\n- Install the DataStore extension"); @@ -263,207 +267,26 @@ rm -rf README ckan-compose ahoy dpp_default_config.ini get-docker.sh permissions success_text(format!("6. Installed CKAN {}.", config.ckan_version).as_str()) ); + // Install extensions + if config.extension_datastore { + step_install_datastore_extension("7.".to_string(), &sh, username.clone())?; + } + if config.extension_ckanext_scheming { + step_install_ckanext_scheming_extension("8.".to_string(), &sh, username.clone())?; + } if config.extension_datapusher_plus { - println!( - "\n{} Enabling DataStore plugin, adding config URLs in /etc/ckan/default/ckan.ini and updating permissions...", - step_text("7."), - ); - let mut conf = ini::Ini::load_from_file("/etc/ckan/default/ckan.ini")?; - let app_main_section = conf.section_mut(Some("app:main")).unwrap(); - let mut ckan_plugins = app_main_section.get("ckan.plugins").unwrap().to_string(); - ckan_plugins.push_str(" datastore"); - app_main_section.insert("ckan.plugins", ckan_plugins); - app_main_section.insert( - "ckan.datastore.write_url", - "postgresql://ckan_default:pass@localhost/datastore_default", - ); - app_main_section.insert( - "ckan.datastore.read_url", - "postgresql://datastore_default:pass@localhost/datastore_default", - ); - app_main_section.insert("ckan.datastore.sqlsearch.enabled", "true"); - conf.write_to_file("/etc/ckan/default/ckan.ini")?; - let postgres_container_id = cmd!( - sh, - "sudo docker ps -aqf name=^ckan-devstaller-project-postgres$" - ) - .read()?; - let set_permissions_output = cmd!( - sh, - "ckan -c /etc/ckan/default/ckan.ini datastore set-permissions" - ) - .read()?; - std::fs::write("permissions.sql", set_permissions_output)?; - loop { - std::thread::sleep(std::time::Duration::from_secs(2)); - if std::fs::exists("permissions.sql")? { - break; - } - } - sh.change_dir(format!("/home/{username}")); - cmd!( - sh, - "sudo docker cp permissions.sql {postgres_container_id}:/permissions.sql" - ) - .run()?; - cmd!(sh, "sudo docker exec {postgres_container_id} psql -U ckan_default --set ON_ERROR_STOP=1 -f permissions.sql").run()?; - println!( - "{}", - success_text( - "7. Enabled DataStore plugin, set DataStore URLs in /etc/ckan/default/ckan.ini, and updated permissions." - ) - ); - - println!( - "{}", - step_text("\n{} Installing ckanext-scheming and DataPusher+ extensions..."), - ); - cmd!( - sh, - "pip install -e git+https://github.com/ckan/ckanext-scheming.git#egg=ckanext-scheming" - ) - .run()?; - let mut conf = ini::Ini::load_from_file("/etc/ckan/default/ckan.ini")?; - let app_main_section = conf.section_mut(Some("app:main")).unwrap(); - let mut ckan_plugins = app_main_section.get("ckan.plugins").unwrap().to_string(); - ckan_plugins.push_str(" scheming_datasets"); - cmd!( - sh, - "ckan config-tool /etc/ckan/default/ckan.ini -s app:main ckan.plugins={ckan_plugins}" - ) - .run()?; - cmd!(sh, "ckan config-tool /etc/ckan/default/ckan.ini -s app:main scheming.presets=ckanext.scheming:presets.json").run()?; - cmd!(sh, "ckan config-tool /etc/ckan/default/ckan.ini -s app:main scheming.dataset_fallback=false").run()?; - // app_main_section.insert("ckan.plugins", ckan_plugins); - // app_main_section.insert("scheming.presets", "ckanext.scheming:presets.json"); - // app_main_section.insert("scheming.dataset_fallback", "false"); - // conf.write_to_file("/etc/ckan/default/ckan.ini")?; - // Install DataPusher+ - cmd!(sh, "sudo apt install python3-virtualenv python3-dev python3-pip python3-wheel build-essential libxslt1-dev libxml2-dev zlib1g-dev git libffi-dev libpq-dev uchardet -y").run()?; - sh.change_dir("/usr/lib/ckan/default/src"); - cmd!(sh, "pip install -e git+https://github.com/dathere/datapusher-plus.git@main#egg=datapusher-plus").run()?; - sh.change_dir("/usr/lib/ckan/default/src/datapusher-plus"); - cmd!(sh, "pip install -r requirements.txt").run()?; - sh.change_dir(format!("/home/{username}")); - cmd!(sh, "wget https://github.com/dathere/qsv/releases/download/4.0.0/qsv-4.0.0-x86_64-unknown-linux-gnu.zip").run()?; - cmd!(sh, "sudo apt install unzip -y").run()?; - cmd!(sh, "unzip qsv-4.0.0-x86_64-unknown-linux-gnu.zip").run()?; - cmd!(sh, "sudo rm -rf qsv-4.0.0-x86_64-unknown-linux-gnu.zip").run()?; - cmd!(sh, "sudo mv ./qsvdp_glibc-2.31 /usr/local/bin/qsvdp").run()?; - let mut conf = ini::Ini::load_from_file("/etc/ckan/default/ckan.ini")?; - let app_main_section = conf.section_mut(Some("app:main")).unwrap(); - let mut ckan_plugins = app_main_section.get("ckan.plugins").unwrap().to_string(); - ckan_plugins.push_str(" datapusher_plus"); - cmd!( - sh, - "ckan config-tool /etc/ckan/default/ckan.ini -s app:main ckan.plugins={ckan_plugins}" - ) - .run()?; - cmd!(sh, "ckan config-tool /etc/ckan/default/ckan.ini -s app:main scheming.dataset_schemas=ckanext.datapusher_plus:dataset-druf.yaml").run()?; - // app_main_section.insert("ckan.plugins", ckan_plugins); - // app_main_section.insert( - // "scheming.dataset_schemas", - // "ckanext.datapusher_plus:dataset-druf.yaml", - // ); - // conf.write_to_file("/etc/ckan/default/ckan.ini")?; - let dpp_default_config = r#" -ckanext.datapusher_plus.use_proxy = false -ckanext.datapusher_plus.download_proxy = -ckanext.datapusher_plus.ssl_verify = false -# supports INFO, DEBUG, TRACE - use DEBUG or TRACE when debugging scheming Formulas -ckanext.datapusher_plus.upload_log_level = INFO -ckanext.datapusher_plus.formats = csv tsv tab ssv xls xlsx xlsxb xlsm ods geojson shp qgis zip -ckanext.datapusher_plus.pii_screening = false -ckanext.datapusher_plus.pii_found_abort = false -ckanext.datapusher_plus.pii_regex_resource_id_or_alias = -ckanext.datapusher_plus.pii_show_candidates = false -ckanext.datapusher_plus.pii_quick_screen = false -ckanext.datapusher_plus.qsv_bin = /usr/local/bin/qsvdp -ckanext.datapusher_plus.preview_rows = 100 -ckanext.datapusher_plus.download_timeout = 300 -ckanext.datapusher_plus.max_content_length = 1256000000000 -ckanext.datapusher_plus.chunk_size = 16384 -ckanext.datapusher_plus.default_excel_sheet = 0 -ckanext.datapusher_plus.sort_and_dupe_check = true -ckanext.datapusher_plus.dedup = false -ckanext.datapusher_plus.unsafe_prefix = unsafe_ -ckanext.datapusher_plus.reserved_colnames = _id -ckanext.datapusher_plus.prefer_dmy = false -ckanext.datapusher_plus.ignore_file_hash = true -ckanext.datapusher_plus.auto_index_threshold = 3 -ckanext.datapusher_plus.auto_index_dates = true -ckanext.datapusher_plus.auto_unique_index = true -ckanext.datapusher_plus.summary_stats_options = -ckanext.datapusher_plus.add_summary_stats_resource = false -ckanext.datapusher_plus.summary_stats_with_preview = false -ckanext.datapusher_plus.qsv_stats_string_max_length = 32767 -ckanext.datapusher_plus.qsv_dates_whitelist = date,time,due,open,close,created -ckanext.datapusher_plus.qsv_freq_limit = 10 -ckanext.datapusher_plus.auto_alias = true -ckanext.datapusher_plus.auto_alias_unique = false -ckanext.datapusher_plus.copy_readbuffer_size = 1048576 -ckanext.datapusher_plus.type_mapping = {"String": "text", "Integer": "numeric","Float": "numeric","DateTime": "timestamp","Date": "date","NULL": "text"} -ckanext.datapusher_plus.auto_spatial_simplication = true -ckanext.datapusher_plus.spatial_simplication_relative_tolerance = 0.1 -ckanext.datapusher_plus.latitude_fields = latitude,lat -ckanext.datapusher_plus.longitude_fields = longitude,long,lon -ckanext.datapusher_plus.jinja2_bytecode_cache_dir = /tmp/jinja2_butecode_cache -ckanext.datapusher_plus.auto_unzip_one_file = true -ckanext.datapusher_plus.api_token = -ckanext.datapusher_plus.describeGPT_api_key = -ckanext.datapusher_plus.file_bin = /usr/bin/file -ckanext.datapusher_plus.enable_druf = false -ckanext.datapusher_plus.enable_form_redirect = true -"#; - std::fs::write("dpp_default_config.ini", dpp_default_config)?; - cmd!( - sh, - "ckan config-tool /etc/ckan/default/ckan.ini -f dpp_default_config.ini" - ) - .run()?; - let resource_formats_str = std::fs::read_to_string( - "/usr/lib/ckan/default/src/ckan/config/resource_formats.json", + step_install_datapusher_plus_extension( + "9.".to_string(), + &sh, + sysadmin_username, + username.clone(), )?; - let mut resource_formats_val: serde_json::Value = - serde_json::from_str(&resource_formats_str)?; - let all_resource_formats = resource_formats_val - .get_mut(0) - .unwrap() - .as_array_mut() - .unwrap(); - all_resource_formats.push(json!([ - "TAB", - "Tab Separated Values File", - "text/tab-separated-values", - [] - ])); - std::fs::write( - "/usr/lib/ckan/default/src/ckan/config/resource_formats.json", - serde_json::to_string(&resource_formats_val)?, - )?; - cmd!(sh, "sudo locale-gen en_US.UTF-8").run()?; - cmd!(sh, "sudo update-locale").run()?; - let token_command_output = cmd!( - sh, - "ckan -c /etc/ckan/default/ckan.ini user token add {sysadmin_username} dpplus" - ) - .read()?; - let tail_output = cmd!(sh, "tail -n 1").stdin(token_command_output).read()?; - let dpp_api_token = cmd!(sh, "tr -d '\t'").stdin(tail_output).read()?; - cmd!(sh, "ckan config-tool /etc/ckan/default/ckan.ini ckanext.datapusher_plus.api_token={dpp_api_token}").env("LC_ALL", "en_US.UTF-8").run()?; - cmd!( - sh, - "ckan -c /etc/ckan/default/ckan.ini db upgrade -p datapusher_plus" - ) - .run()?; - println!( - "{}", - success_text("8. Installed ckanext-scheming and DataPusher+ extensions.") - ); } - println!("\n{}", success_text("Running CKAN instance...")); - cmd!(sh, "ckan -c /etc/ckan/default/ckan.ini run").run()?; + if !args.skip_run { + println!("\n{}", success_text("Running CKAN instance...")); + cmd!(sh, "ckan -c /etc/ckan/default/ckan.ini run").run()?; + } } else { println!("Cancelling installation."); } diff --git a/src/questions.rs b/src/questions.rs index f8d5122..402639e 100644 --- a/src/questions.rs +++ b/src/questions.rs @@ -16,7 +16,7 @@ pub fn question_ssh() -> Result { } pub fn question_ckan_version() -> Result { - let ckan_version_options: Vec<&str> = vec!["2.11.3", "2.10.8", "Other"]; + let ckan_version_options: Vec<&str> = vec!["2.11.4", "2.10.9", "Other"]; let answer_ckan_version = Select::new( "What CKAN version would you like to install? (optional)", ckan_version_options, @@ -26,7 +26,7 @@ pub fn question_ckan_version() -> Result { if answer_ckan_version == "Other" { Ok( Text::new("What CKAN version would you like to install? (optional)") - .with_default("2.11.3") + .with_default("2.11.4") .prompt()?, ) } else { diff --git a/src/steps.rs b/src/steps.rs index 1cc0d3c..af19091 100644 --- a/src/steps.rs +++ b/src/steps.rs @@ -1,5 +1,6 @@ use crate::styles::{highlighted_text, important_text, step_text, success_text}; use anyhow::Result; +use serde_json::json; use xshell::{Shell, cmd}; pub fn step_intro() { @@ -10,7 +11,7 @@ pub fn step_intro() { ); println!( "This installer should assist in setting up {} from a source installation along with ckan-compose. If you have any issues, please report them at https://support.dathere.com or https://github.com/dathere/ckan-devstaller/issues.", - highlighted_text("CKAN 2.11.3") + highlighted_text("CKAN 2.11.4") ); println!( "\nYou may also learn more about ckan-devstaller at https://ckan-devstaller.dathere.com." @@ -135,3 +136,252 @@ POSTGRES_PASSWORD=pass"; ); Ok(()) } + +pub fn step_install_datastore_extension( + step_prefix: String, + sh: &Shell, + username: String, +) -> Result<()> { + println!( + "\n{} Enabling DataStore plugin, adding config URLs in /etc/ckan/default/ckan.ini and updating permissions...", + step_text(step_prefix.as_str()), + ); + let mut conf = ini::Ini::load_from_file("/etc/ckan/default/ckan.ini")?; + let app_main_section = conf.section_mut(Some("app:main")).unwrap(); + let mut ckan_plugins = app_main_section.get("ckan.plugins").unwrap().to_string(); + ckan_plugins.push_str(" datastore"); + app_main_section.insert("ckan.plugins", ckan_plugins); + app_main_section.insert( + "ckan.datastore.write_url", + "postgresql://ckan_default:pass@localhost/datastore_default", + ); + app_main_section.insert( + "ckan.datastore.read_url", + "postgresql://datastore_default:pass@localhost/datastore_default", + ); + app_main_section.insert("ckan.datastore.sqlsearch.enabled", "true"); + conf.write_to_file("/etc/ckan/default/ckan.ini")?; + let postgres_container_id = cmd!( + sh, + "sudo docker ps -aqf name=^ckan-devstaller-project-postgres$" + ) + .read()?; + let set_permissions_output = cmd!( + sh, + "ckan -c /etc/ckan/default/ckan.ini datastore set-permissions" + ) + .read()?; + std::fs::write("permissions.sql", set_permissions_output)?; + loop { + std::thread::sleep(std::time::Duration::from_secs(2)); + if std::fs::exists("permissions.sql")? { + break; + } + } + sh.change_dir(format!("/home/{username}")); + cmd!( + sh, + "sudo docker cp permissions.sql {postgres_container_id}:/permissions.sql" + ) + .run()?; + cmd!(sh, "sudo docker exec {postgres_container_id} psql -U ckan_default --set ON_ERROR_STOP=1 -f permissions.sql").run()?; + println!( + "{}", + success_text( + format!("{step_prefix} Enabled DataStore plugin, set DataStore URLs in /etc/ckan/default/ckan.ini, and updated permissions.").as_str() + ) + ); + Ok(()) +} + +pub fn step_install_ckanext_scheming_extension( + step_prefix: String, + sh: &Shell, + username: String, +) -> Result<()> { + println!( + "{}", + step_text("\n{} Installing the ckanext-scheming extension..."), + ); + cmd!( + sh, + "pip install -e git+https://github.com/ckan/ckanext-scheming.git#egg=ckanext-scheming" + ) + .run()?; + let mut conf = ini::Ini::load_from_file("/etc/ckan/default/ckan.ini")?; + let app_main_section = conf.section_mut(Some("app:main")).unwrap(); + let mut ckan_plugins = app_main_section.get("ckan.plugins").unwrap().to_string(); + ckan_plugins.push_str(" scheming_datasets"); + cmd!( + sh, + "ckan config-tool /etc/ckan/default/ckan.ini -s app:main ckan.plugins={ckan_plugins}" + ) + .run()?; + cmd!(sh, "ckan config-tool /etc/ckan/default/ckan.ini -s app:main scheming.presets=ckanext.scheming:presets.json").run()?; + cmd!( + sh, + "ckan config-tool /etc/ckan/default/ckan.ini -s app:main scheming.dataset_fallback=false" + ) + .run()?; + // app_main_section.insert("ckan.plugins", ckan_plugins); + // app_main_section.insert("scheming.presets", "ckanext.scheming:presets.json"); + // app_main_section.insert("scheming.dataset_fallback", "false"); + // conf.write_to_file("/etc/ckan/default/ckan.ini")?; + Ok(()) +} + +pub fn step_install_datapusher_plus_extension( + step_prefix: String, + sh: &Shell, + sysadmin_username: String, + username: String, +) -> Result<()> { + // Install DataPusher+ + println!( + "{}", + step_text(format!("\n{step_prefix} Installing DataPusher+ extension...").as_str()) + ); + cmd!(sh, "sudo apt install python3-virtualenv python3-dev python3-pip python3-wheel build-essential libxslt1-dev libxml2-dev zlib1g-dev git libffi-dev libpq-dev uchardet -y").run()?; + sh.change_dir("/usr/lib/ckan/default/src"); + cmd!( + sh, + "pip install -e git+https://github.com/dathere/datapusher-plus.git@main#egg=datapusher-plus" + ) + .run()?; + sh.change_dir("/usr/lib/ckan/default/src/datapusher-plus"); + cmd!(sh, "pip install -r requirements.txt").run()?; + sh.change_dir(format!("/home/{username}")); + cmd!(sh, "wget https://github.com/dathere/qsv/releases/download/4.0.0/qsv-4.0.0-x86_64-unknown-linux-gnu.zip").run()?; + cmd!(sh, "sudo apt install unzip -y").run()?; + cmd!(sh, "unzip qsv-4.0.0-x86_64-unknown-linux-gnu.zip").run()?; + cmd!(sh, "sudo rm -rf qsv-4.0.0-x86_64-unknown-linux-gnu.zip").run()?; + cmd!(sh, "sudo mv ./qsvdp_glibc-2.31 /usr/local/bin/qsvdp").run()?; + let mut conf = ini::Ini::load_from_file("/etc/ckan/default/ckan.ini")?; + let app_main_section = conf.section_mut(Some("app:main")).unwrap(); + let mut ckan_plugins = app_main_section.get("ckan.plugins").unwrap().to_string(); + ckan_plugins.push_str(" datapusher_plus"); + app_main_section.insert("ckan.plugins", ckan_plugins); + app_main_section.insert( + "scheming.dataset_schemas", + "ckanext.datapusher_plus:dataset-druf.yaml", + ); + app_main_section.insert("ckanext.datapusher_plus.use_proxy", "false"); + app_main_section.insert("ckanext.datapusher_plus.download_proxy", ""); + app_main_section.insert("ckanext.datapusher_plus.ssl_verify", "false"); + app_main_section.insert("ckanext.datapusher_plus.upload_log_level", "INFO"); + app_main_section.insert( + "ckanext.datapusher_plus.formats", + "csv tsv tab ssv xls xlsx xlsxb xlsm ods geojson shp qgis zip", + ); + app_main_section.insert("ckanext.datapusher_plus.pii_screening", "false"); + app_main_section.insert("ckanext.datapusher_plus.pii_found_abort", "false"); + app_main_section.insert("ckanext.datapusher_plus.pii_regex_resource_id_or_alias", ""); + app_main_section.insert("ckanext.datapusher_plus.pii_show_candidates", "false"); + app_main_section.insert("ckanext.datapusher_plus.pii_quick_screen", "false"); + app_main_section.insert("ckanext.datapusher_plus.qsv_bin", "/usr/local/bin/qsvdp"); + app_main_section.insert("ckanext.datapusher_plus.preview_rows", "100"); + app_main_section.insert("ckanext.datapusher_plus.download_timeout", "300"); + app_main_section.insert( + "ckanext.datapusher_plus.max_content_length", + "1256000000000", + ); + app_main_section.insert("ckanext.datapusher_plus.chunk_size", "16384"); + app_main_section.insert("ckanext.datapusher_plus.default_excel_sheet", "0"); + app_main_section.insert("ckanext.datapusher_plus.sort_and_dupe_check", "true"); + app_main_section.insert("ckanext.datapusher_plus.dedup", "false"); + app_main_section.insert("ckanext.datapusher_plus.unsafe_prefix", "unsafe_"); + app_main_section.insert("ckanext.datapusher_plus.reserved_colnames", "_id"); + app_main_section.insert("ckanext.datapusher_plus.prefer_dmy", "false"); + app_main_section.insert("ckanext.datapusher_plus.ignore_file_hash", "true"); + app_main_section.insert("ckanext.datapusher_plus.auto_index_threshold", "3"); + app_main_section.insert("ckanext.datapusher_plus.auto_index_dates", "true"); + app_main_section.insert("ckanext.datapusher_plus.auto_unique_index", "true"); + app_main_section.insert("ckanext.datapusher_plus.summary_stats_options", ""); + app_main_section.insert( + "ckanext.datapusher_plus.add_summary_stats_resource", + "false", + ); + app_main_section.insert( + "ckanext.datapusher_plus.summary_stats_with_preview", + "false", + ); + app_main_section.insert( + "ckanext.datapusher_plus.qsv_stats_string_max_length", + "32767", + ); + app_main_section.insert( + "ckanext.datapusher_plus.qsv_dates_whitelist", + "date,time,due,open,close,created", + ); + app_main_section.insert("ckanext.datapusher_plus.qsv_freq_limit", "10"); + app_main_section.insert("ckanext.datapusher_plus.auto_alias", "true"); + app_main_section.insert("ckanext.datapusher_plus.auto_alias_unique", "false"); + app_main_section.insert("ckanext.datapusher_plus.copy_readbuffer_size", "1048576"); + app_main_section.insert("ckanext.datapusher_plus.type_mapping", r#"{"String": "text", "Integer": "numeric","Float": "numeric","DateTime": "timestamp","Date": "date","NULL": "text"}"#); + app_main_section.insert("ckanext.datapusher_plus.auto_spatial_simplication", "true"); + app_main_section.insert( + "ckanext.datapusher_plus.spatial_simplication_relative_tolerance", + "0.1", + ); + app_main_section.insert("ckanext.datapusher_plus.latitude_fields", "latitude,lat"); + app_main_section.insert( + "ckanext.datapusher_plus.longitude_fields", + "longitude,long,lon", + ); + app_main_section.insert( + "ckanext.datapusher_plus.jinja2_bytecode_cache_dir", + "/tmp/jinja2_butecode_cache", + ); + app_main_section.insert("ckanext.datapusher_plus.auto_unzip_one_file", "true"); + app_main_section.insert( + "ckanext.datapusher_plus.api_token", + "", + ); + app_main_section.insert( + "ckanext.datapusher_plus.describeGPT_api_key", + "", + ); + app_main_section.insert("ckanext.datapusher_plus.file_bin", "/usr/bin/file"); + app_main_section.insert("ckanext.datapusher_plus.enable_druf", "false"); + app_main_section.insert("ckanext.datapusher_plus.enable_form_redirect", "true"); + conf.write_to_file("/etc/ckan/default/ckan.ini")?; + let resource_formats_str = std::fs::read_to_string( + "/usr/lib/ckan/default/src/ckan/ckan/config/resource_formats.json", + )?; + let mut resource_formats_val: serde_json::Value = serde_json::from_str(&resource_formats_str)?; + let all_resource_formats = resource_formats_val + .get_mut(0) + .unwrap() + .as_array_mut() + .unwrap(); + all_resource_formats.push(json!([ + "TAB", + "Tab Separated Values File", + "text/tab-separated-values", + [] + ])); + std::fs::write( + "/usr/lib/ckan/default/src/ckan/ckan/config/resource_formats.json", + serde_json::to_string(&resource_formats_val)?, + )?; + cmd!(sh, "sudo locale-gen en_US.UTF-8").run()?; + cmd!(sh, "sudo update-locale").run()?; + let token_command_output = cmd!( + sh, + "ckan -c /etc/ckan/default/ckan.ini user token add {sysadmin_username} dpplus" + ) + .read()?; + let tail_output = cmd!(sh, "tail -n 1").stdin(token_command_output).read()?; + let dpp_api_token = cmd!(sh, "tr -d '\t'").stdin(tail_output).read()?; + cmd!(sh, "ckan config-tool /etc/ckan/default/ckan.ini ckanext.datapusher_plus.api_token={dpp_api_token}").env("LC_ALL", "en_US.UTF-8").run()?; + cmd!( + sh, + "ckan -c /etc/ckan/default/ckan.ini db upgrade -p datapusher_plus" + ) + .run()?; + println!( + "{}", + success_text(format!("{step_prefix} Installed DataPusher+ extension.").as_str()) + ); + Ok(()) +}