) => void;
+ actionTest: () => void;
+ continueId: string;
+ resetTest: () => void;
+ setValidationSucceeded: DataConnectorModalContinueButtonProps["setValidationSucceeded"];
+ step: number;
+ testId: string;
+ testIsFailure: boolean;
+ testIsOngoing: boolean;
+ testIsSuccess: boolean;
+}
+function TestConnectionAndContinueButtons({
+ actionState,
+ actionTest,
+ continueId,
+ resetTest,
+ setValidationSucceeded,
+ step,
+ testId,
+ testIsFailure,
+ testIsOngoing,
+ testIsSuccess,
+}: TestConnectionAndContinueButtonsProps) {
+ const buttonTestId = `${testId}-button`;
+ const divTestId = `${testId}-div`;
+ const testConnectionContent =
+ testIsSuccess || testIsFailure ? (
+ <>
+ Re-test
+ actionTest()}
+ >
+ {testConnectionContent}
+
+
+ );
+
+ const buttonContinueId = `${continueId}-button`;
+ const divContinueId = `${continueId}-div`;
+ const continueContent = testIsSuccess ? (
+ <>
+ Continue
+ {
+ setValidationSucceeded(testIsSuccess);
+ if (testIsFailure || testIsSuccess) {
+ resetTest();
+ }
+ actionState({
+ step: step === 0 ? CLOUD_STORAGE_TOTAL_STEPS : step + 1,
+ completedSteps: step === 0 ? CLOUD_STORAGE_TOTAL_STEPS - 1 : step,
+ });
+ }}
+ >
+ {continueContent}
+
+ {testIsFailure && (
+
+ The connection is not working as configured. You can make changes
+ and try again, or skip and continue.
+
+ )}
+
+ );
+
+ return (
+
+ {testConnectionSection}
+ {continueSection}
+
+ );
+}
diff --git a/client/src/features/dataConnectorsV2/components/DataConnectorModal/index.tsx b/client/src/features/dataConnectorsV2/components/DataConnectorModal/index.tsx
new file mode 100644
index 0000000000..79db65af68
--- /dev/null
+++ b/client/src/features/dataConnectorsV2/components/DataConnectorModal/index.tsx
@@ -0,0 +1,521 @@
+/*!
+ * Copyright 2024 - Swiss Data Science Center (SDSC)
+ * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+ * Eidgenössische Technische Hochschule Zürich (ETHZ).
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { skipToken } from "@reduxjs/toolkit/query";
+import cx from "classnames";
+import { isEqual } from "lodash-es";
+import { useCallback, useEffect, useMemo, useState } from "react";
+import { ArrowCounterclockwise, Database } from "react-bootstrap-icons";
+import { Button, Modal, ModalBody, ModalFooter, ModalHeader } from "reactstrap";
+
+import { RtkOrNotebooksError } from "../../../../components/errors/RtkErrorAlert";
+
+import AddStorageBreadcrumbNavbar from "../../../project/components/cloudStorage/AddStorageBreadcrumbNavbar";
+import {
+ useGetCloudStorageSchemaQuery,
+ useTestCloudStorageConnectionMutation,
+} from "../../../project/components/cloudStorage/projectCloudStorage.api";
+import {
+ CLOUD_STORAGE_TOTAL_STEPS,
+ EMPTY_CLOUD_STORAGE_STATE,
+} from "../../../project/components/cloudStorage/projectCloudStorage.constants";
+import {
+ AddCloudStorageState,
+ CloudStorageDetailsOptions,
+ TestCloudStorageConnectionParams,
+} from "../../../project/components/cloudStorage/projectCloudStorage.types";
+
+import {
+ findSensitive,
+ getSchemaProviders,
+ hasProviderShortlist,
+} from "../../../project/utils/projectCloudStorage.utils";
+
+import { usePostStoragesV2ByStorageIdSecretsMutation } from "../../../projectsV2/api/projectV2.enhanced-api";
+import {
+ usePatchDataConnectorsByDataConnectorIdMutation,
+ usePostDataConnectorsMutation,
+} from "../../../projectsV2/api/data-connectors.enhanced-api";
+import type { DataConnectorRead } from "../../../projectsV2/api/data-connectors.api";
+
+import styles from "./DataConnectorModal.module.scss";
+
+import {
+ DataConnectorModalBackButton,
+ DataConnectorModalContinueButton,
+ DataConnectorConnectionTestResult,
+} from "./dataConnectorModalButtons";
+import DataConnectorModalBody from "./DataConnectorModalBody";
+import type { CredentialSaveStatus } from "./DataConnectorModalResult";
+import {
+ dataConnectorPostFromFlattened,
+ dataConnectorToFlattened,
+ EMPTY_DATA_CONNECTOR_FLAT,
+ type DataConnectorFlat,
+} from "../dataConnector.utils";
+
+interface DataConnectorModalProps {
+ dataConnector?: DataConnectorRead | null;
+ isOpen: boolean;
+ namespace: string;
+ toggle: () => void;
+}
+export default function DataConnectorModal({
+ dataConnector = null,
+ isOpen,
+ namespace,
+ toggle: originalToggle,
+}: DataConnectorModalProps) {
+ const dataConnectorId = dataConnector?.id ?? null;
+ // Fetch available schema when users open the modal
+ const schemaQueryResult = useGetCloudStorageSchemaQuery(
+ isOpen ? undefined : skipToken
+ );
+ const { data: schema } = schemaQueryResult;
+
+ // Reset state on props change
+ useEffect(() => {
+ const flattened = dataConnectorToFlattened(dataConnector);
+ const cloudStorageState: AddCloudStorageState =
+ dataConnector != null
+ ? {
+ ...EMPTY_CLOUD_STORAGE_STATE,
+ step: 2,
+ completedSteps: CLOUD_STORAGE_TOTAL_STEPS,
+ }
+ : EMPTY_CLOUD_STORAGE_STATE;
+ setFlatDataConnector(flattened);
+ setState(cloudStorageState);
+ }, [dataConnector]);
+
+ const [success, setSuccess] = useState(false);
+ const [credentialSaveStatus, setCredentialSaveStatus] =
+ useState("none");
+ const [validationSucceeded, setValidationSucceeded] = useState(false);
+ const [state, setState] = useState(
+ EMPTY_CLOUD_STORAGE_STATE
+ );
+ const initialFlatDataConnector = EMPTY_DATA_CONNECTOR_FLAT;
+ initialFlatDataConnector.namespace = namespace;
+ const [flatDataConnector, setFlatDataConnector] = useState(
+ initialFlatDataConnector
+ );
+
+ // Enhanced setters
+ const setStateSafe = useCallback(
+ (newState: Partial) => {
+ const fullNewState = {
+ ...state,
+ ...newState,
+ };
+ if (isEqual(fullNewState, state)) {
+ return;
+ }
+
+ // Handle advanced mode changes
+ if (
+ fullNewState.advancedMode !== state.advancedMode &&
+ fullNewState.step !== 3
+ ) {
+ if (fullNewState.advancedMode) {
+ fullNewState.step = 0;
+ } else {
+ if (
+ // schema and provider (where necessary) must also exist in the list
+ !flatDataConnector.schema ||
+ !schema?.find((s) => s.prefix === flatDataConnector.schema) ||
+ (hasProviderShortlist(flatDataConnector.schema) &&
+ (!flatDataConnector.provider ||
+ !getSchemaProviders(
+ schema,
+ false,
+ flatDataConnector.schema
+ )?.find((p) => p.name === flatDataConnector.provider)))
+ ) {
+ fullNewState.step = 1;
+ } else {
+ fullNewState.step = 2;
+ }
+ }
+ }
+ setState(fullNewState);
+ },
+ [state, flatDataConnector, schema]
+ );
+
+ // Reset
+ const [redraw, setRedraw] = useState(false);
+ useEffect(() => {
+ if (redraw) setRedraw(false);
+ }, [redraw]);
+
+ // Mutations
+ const [createDataConnector, createResult] = usePostDataConnectorsMutation();
+ const [updateDataConnector, updateResult] =
+ usePatchDataConnectorsByDataConnectorIdMutation();
+ const [saveCredentials, saveCredentialsResult] =
+ usePostStoragesV2ByStorageIdSecretsMutation();
+ const [validateCloudStorageConnection, validationResult] =
+ useTestCloudStorageConnectionMutation();
+
+ const reset = useCallback(() => {
+ const resetStatus = dataConnectorToFlattened(dataConnector);
+ setState((prevState) =>
+ dataConnector != null
+ ? {
+ ...EMPTY_CLOUD_STORAGE_STATE,
+ step: prevState.step,
+ completedSteps: prevState.completedSteps,
+ }
+ : {
+ ...EMPTY_CLOUD_STORAGE_STATE,
+ }
+ );
+ createResult.reset();
+ validationResult.reset();
+ setFlatDataConnector(resetStatus);
+ setSuccess(false);
+ setCredentialSaveStatus("none");
+ setValidationSucceeded(false);
+ setRedraw(true); // This forces re-loading the useForm fields
+ }, [createResult, dataConnector, validationResult]);
+
+ const setFlatDataConnectorSafe = useCallback(
+ (newDataConnector: Partial) => {
+ const fullNewDetails = {
+ ...flatDataConnector,
+ ...newDataConnector,
+ };
+ if (isEqual(fullNewDetails, flatDataConnector)) {
+ return;
+ }
+ // reset follow-up properties: schema > provider > options
+ if (fullNewDetails.schema !== flatDataConnector.schema) {
+ fullNewDetails.provider = undefined;
+ fullNewDetails.options = undefined;
+ fullNewDetails.sourcePath = undefined;
+ } else if (fullNewDetails.provider !== flatDataConnector.provider) {
+ fullNewDetails.options = undefined;
+ fullNewDetails.sourcePath = undefined;
+ }
+ if (!validationResult.isUninitialized) validationResult.reset();
+ setFlatDataConnector(fullNewDetails);
+ },
+ [flatDataConnector, validationResult]
+ );
+
+ const validateConnection = useCallback(() => {
+ const validateParameters: TestCloudStorageConnectionParams = {
+ configuration: {
+ type: flatDataConnector.schema,
+ },
+ source_path: flatDataConnector.sourcePath ?? "/",
+ };
+ if (flatDataConnector.provider) {
+ validateParameters.configuration.provider = flatDataConnector.provider;
+ }
+ if (
+ flatDataConnector.options &&
+ Object.keys(flatDataConnector.options).length > 0
+ ) {
+ const options = flatDataConnector.options as CloudStorageDetailsOptions;
+ Object.entries(options).forEach(([key, value]) => {
+ if (value != undefined && value !== "") {
+ validateParameters.configuration[key] = value;
+ }
+ });
+ }
+
+ validateCloudStorageConnection(validateParameters);
+ }, [flatDataConnector, validateCloudStorageConnection]);
+
+ const addOrEditStorage = useCallback(() => {
+ const dataConnectorPost = dataConnectorPostFromFlattened(
+ flatDataConnector,
+ schema ?? [],
+ dataConnector
+ );
+
+ // We manually set success only when we get an ID back. That's just to show a success message
+ if (dataConnector && dataConnectorId) {
+ updateDataConnector({
+ dataConnectorId,
+ dataConnectorPatch: dataConnectorPost,
+ "If-Match": dataConnector.etag,
+ }).then((result) => {
+ if ("data" in result && result.data.id) {
+ setSuccess(true);
+ }
+ });
+ } else {
+ createDataConnector({
+ dataConnectorPost,
+ }).then((result) => {
+ if ("data" in result && result.data.id) {
+ setSuccess(true);
+ }
+ });
+ }
+ }, [
+ createDataConnector,
+ dataConnector,
+ dataConnectorId,
+ updateDataConnector,
+ schema,
+ flatDataConnector,
+ ]);
+
+ const toggle = useCallback(() => {
+ originalToggle();
+ setCredentialSaveStatus("none");
+ setValidationSucceeded(false);
+ if (success) {
+ setSuccess(false);
+ reset();
+ } else {
+ createResult.reset();
+ validationResult.reset();
+ }
+ }, [createResult, originalToggle, reset, success, validationResult]);
+
+ // Handle unmount
+ useEffect(() => {
+ const cleanup = () => {
+ reset();
+ };
+
+ return cleanup;
+ }, []); // eslint-disable-line react-hooks/exhaustive-deps
+
+ const schemaRequiresProvider = useMemo(
+ () => hasProviderShortlist(flatDataConnector.schema),
+ [flatDataConnector.schema]
+ );
+
+ useEffect(() => {
+ const dataConnectorId = createResult.data?.id;
+ if (dataConnectorId == null) return;
+ const shouldSaveCredentials = shouldSaveDataConnectorCredentials(
+ flatDataConnector.options,
+ state.saveCredentials,
+ validationSucceeded
+ );
+ if (!shouldSaveCredentials) {
+ return;
+ }
+ const options = flatDataConnector.options as CloudStorageDetailsOptions;
+ if (!schema) return;
+ const sensitiveFieldNames = findSensitive(
+ schema.find((s) => s.prefix === flatDataConnector.schema)
+ );
+ const cloudStorageSecretPostList = sensitiveFieldNames
+ .map((name) => ({
+ name,
+ value: options[name],
+ }))
+ .filter((secret) => secret.value != undefined && secret.value != "")
+ .map((secret) => ({
+ name: secret.name,
+ value: "" + secret.value,
+ }));
+ saveCredentials({
+ storageId: dataConnectorId,
+ cloudStorageSecretPostList,
+ });
+ }, [
+ createResult.data?.id,
+ saveCredentials,
+ state.saveCredentials,
+ schema,
+ flatDataConnector.options,
+ flatDataConnector.schema,
+ validationSucceeded,
+ ]);
+
+ useEffect(() => {
+ const status = !validationSucceeded
+ ? "none"
+ : createResult.data?.id == null || saveCredentialsResult.isUninitialized
+ ? "none"
+ : saveCredentialsResult.isLoading
+ ? "trying"
+ : saveCredentialsResult.isSuccess
+ ? "success"
+ : saveCredentialsResult.isError
+ ? "failure"
+ : "none";
+ setCredentialSaveStatus(status);
+ }, [createResult, saveCredentialsResult, validationSucceeded]);
+
+ // Visual elements
+ const disableContinueButton =
+ state.step === 1 &&
+ (!flatDataConnector.schema ||
+ (schemaRequiresProvider && !flatDataConnector.provider));
+
+ const isAddResultLoading = createResult.isLoading;
+ const isModifyResultLoading = updateResult.isLoading;
+ const actionError = createResult.error || updateResult.error;
+ const dataConnectorResultNamespace =
+ createResult?.data?.namespace || updateResult?.data?.namespace;
+ const dataConnectorResultSlug =
+ createResult?.data?.slug || updateResult?.data?.slug;
+ const dataConnectorResultName = `${dataConnectorResultNamespace}/${dataConnectorResultSlug}`;
+
+ const disableAddButton =
+ isAddResultLoading ||
+ isModifyResultLoading ||
+ !flatDataConnector.name ||
+ !flatDataConnector.mountPoint ||
+ !flatDataConnector.schema ||
+ (hasProviderShortlist(flatDataConnector.schema) &&
+ !flatDataConnector.provider);
+ const addButtonDisableReason = isAddResultLoading
+ ? "Please wait, the storage is being added"
+ : updateResult.isLoading
+ ? "Please wait, the storage is being modified"
+ : !flatDataConnector.name
+ ? "Please provide a name"
+ : !flatDataConnector.mountPoint
+ ? "Please provide a mount point"
+ : !flatDataConnector.schema
+ ? "Please go back and select a storage type"
+ : "Please go back and select a provider";
+ const isResultLoading = isAddResultLoading || isModifyResultLoading;
+
+ const storageSecrets =
+ dataConnector != null && "secrets" in dataConnector
+ ? dataConnector.secrets ?? []
+ : [];
+ const hasStoredCredentialsInConfig = storageSecrets.length > 0;
+
+ return (
+
+
+
+
+
+
+
+
+
+
+ )}
+
+ {!isResultLoading && !success && (
+ {
+ reset();
+ }}
+ >
+
+ )}
+ {!isResultLoading && (
+
+
+ );
+}
+
+interface DataConnectorModalHeaderProps {
+ dataConnectorId: string | null;
+}
+export function DataConnectorModalHeader({
+ dataConnectorId,
+}: DataConnectorModalHeaderProps) {
+ return (
+ <>
+ , s) => {
+ acc[storageSecretNameToFieldName(s)] = s.name;
+ return acc;
+ }, {}) ?? {};
+ const credentialFieldDefinitions = useMemo(
+ () => getCredentialFieldDefinitions(dataConnector),
+ [dataConnector]
+ );
+ const requiredCredentials = useMemo(
+ () =>
+ credentialFieldDefinitions?.filter((field) => field.requiredCredential),
+ [credentialFieldDefinitions]
+ );
+ const nonRequiredCredentialConfigurationKeys = Object.keys(
+ storageDefinition.configuration
+ ).filter((k) => !requiredCredentials?.some((f) => f.name === k));
+
+ return (
+
+
+
+
+
+
+
+
+ {dataConnector.name}
+
+
+
+
+
+
Data source
+
+
+
+ Mount point {"("}this is where the storage will be mounted during
+ sessions{")"}
+
+
{storageDefinition.target_path}
+
+
+
{storageDefinition.source_path}
+
+
Requires credentials
+
{anySensitiveField ? "Yes" : "No"}
+
+
Required credentials
+
+
+
+ Field
+ Value
+
+
+
+ {requiredCredentials.map(({ name, help }, index) => {
+ const value =
+ name == null
+ ? "unknown"
+ : savedCredentialFields[name]
+ ? CLOUD_STORAGE_SAVED_SECRET_DISPLAY_VALUE
+ : storageDefinition.configuration[name]?.toString();
+ return (
+
+
+ {name}
+ {help &&
+ {value}
+
+ );
+ })}
+
+
+
+
Access mode
+
+ {storageDefinition.readonly
+ ? "Force Read-only"
+ : "Allow Read-Write (requires adequate privileges on the storage)"}
+
+
+
+
+ );
+}
diff --git a/client/src/features/dataConnectorsV2/components/DataConnectorsBox.tsx b/client/src/features/dataConnectorsV2/components/DataConnectorsBox.tsx
new file mode 100644
index 0000000000..992df2e71f
--- /dev/null
+++ b/client/src/features/dataConnectorsV2/components/DataConnectorsBox.tsx
@@ -0,0 +1,395 @@
+/*!
+ * Copyright 2024 - Swiss Data Science Center (SDSC)
+ * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+ * Eidgenössische Technische Hochschule Zürich (ETHZ).
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+
+import cx from "classnames";
+import { useCallback, useEffect, useMemo, useState } from "react";
+import { Database, Globe2, Lock, PlusLg } from "react-bootstrap-icons";
+import { useSearchParams } from "react-router-dom-v5-compat";
+import {
+ Badge,
+ Button,
+ Card,
+ CardBody,
+ CardHeader,
+ Col,
+ ListGroup,
+ ListGroupItem,
+ Row,
+} from "reactstrap";
+
+import ClampedParagraph from "../../../components/clamped/ClampedParagraph";
+import { Loader } from "../../../components/Loader";
+import Pagination from "../../../components/Pagination";
+import { TimeCaption } from "../../../components/TimeCaption";
+import { RtkOrNotebooksError } from "../../../components/errors/RtkErrorAlert";
+import MembershipGuard from "../../ProjectPageV2/utils/MembershipGuard";
+import type { NamespaceKind } from "../../projectsV2/api/namespace.api";
+import type { DataConnector } from "../../projectsV2/api/data-connectors.api";
+import { useGetGroupsByGroupSlugMembersQuery } from "../../projectsV2/api/projectV2.enhanced-api";
+import {
+ useGetDataConnectorsQuery,
+ type GetDataConnectorsApiResponse,
+} from "../../projectsV2/api/data-connectors.enhanced-api";
+import { useGetUserQuery } from "../../user/dataServicesUser.api";
+
+import DataConnectorModal from "./DataConnectorModal";
+import DataConnectorView from "./DataConnectorView";
+
+const DEFAULT_PER_PAGE = 12;
+const DEFAULT_PAGE_PARAM = "page";
+
+function AddButtonForGroupNamespace({
+ namespace,
+ toggleOpen,
+}: Pick) {
+ const { data: members } = useGetGroupsByGroupSlugMembersQuery({
+ groupSlug: namespace,
+ });
+ return (
+
+ ) {
+ const { data: currentUser } = useGetUserQuery();
+
+ if (currentUser?.username === namespace) {
+ return (
+
+
+ );
+ }
+ return null;
+}
+
+interface DataConnectorListDisplayProps {
+ namespace: string;
+ namespaceKind: NamespaceKind;
+ pageParam?: string;
+ perPage?: number;
+}
+
+export default function DataConnectorsBox({
+ namespace: ns,
+ namespaceKind,
+ pageParam: pageParam_,
+ perPage: perPage_,
+}: DataConnectorListDisplayProps) {
+ const pageParam = useMemo(
+ () => (pageParam_ ? pageParam_ : DEFAULT_PAGE_PARAM),
+ [pageParam_]
+ );
+ const perPage = useMemo(
+ () => (perPage_ ? perPage_ : DEFAULT_PER_PAGE),
+ [perPage_]
+ );
+
+ const [searchParams, setSearchParams] = useSearchParams();
+ const onPageChange = useCallback(
+ (pageNumber: number) => {
+ setSearchParams((prevParams) => {
+ if (pageNumber == 1) {
+ prevParams.delete(pageParam);
+ } else {
+ prevParams.set(pageParam, `${pageNumber}`);
+ }
+ return prevParams;
+ });
+ },
+ [pageParam, setSearchParams]
+ );
+
+ const page = useMemo(() => {
+ const pageRaw = searchParams.get(pageParam);
+ if (!pageRaw) {
+ return 1;
+ }
+ try {
+ const page = parseInt(pageRaw, 10);
+ return page > 0 ? page : 1;
+ } catch {
+ return 1;
+ }
+ }, [pageParam, searchParams]);
+
+ const { data, error, isLoading } = useGetDataConnectorsQuery({
+ params: {
+ namespace: ns,
+ page,
+ per_page: perPage,
+ },
+ });
+
+ useEffect(() => {
+ if (data?.totalPages && page > data.totalPages) {
+ setSearchParams(
+ (prevParams) => {
+ if (data.totalPages == 1) {
+ prevParams.delete(pageParam);
+ } else {
+ prevParams.set(pageParam, `${data.totalPages}`);
+ }
+ return prevParams;
+ },
+ { replace: true }
+ );
+ }
+ }, [data?.totalPages, page, pageParam, setSearchParams]);
+
+ if (isLoading) return
+
+
+ {data.total === 0 && (
+
+ Add published datasets from data repositories, and connect to
+ cloud storage to read and write custom data.
+
+ )}
+
+ {data.dataConnectors?.map((dc) => (
+
+
+
+
+
+
+
+
+
+ {totalConnectors}
+
+
+ {namespaceKind === "group" ? (
+
+ ) : (
+
+ )}
+
+
+ );
+}
+
+function DataConnectorLoadingBoxContent() {
+ return (
+
+
+
+
+
+ Retrieving data connectors...
+
+
+ );
+}
+
+interface DataConnectorDisplayProps {
+ dataConnector: DataConnector;
+}
+function DataConnectorDisplay({ dataConnector }: DataConnectorDisplayProps) {
+ const {
+ name,
+ description,
+ visibility,
+ creation_date: creationDate,
+ } = dataConnector;
+
+ const [showDetails, setShowDetails] = useState(false);
+ const toggleDetails = useCallback(() => {
+ setShowDetails((open) => !open);
+ }, []);
+
+ return (
+ <>
+
+
+ {name}
+ {description && {description} }
+
+
+ {visibility.toLowerCase() === "private" ? (
+ <>
+
+
+
+
+ ((options, key) => {
+ const value = allOptions[key];
+ if (value != undefined && value !== "") {
+ options[key] = sensitiveFields.includes(key)
+ ? CLOUD_STORAGE_SENSITIVE_FIELD_TOKEN
+ : value;
+ }
+ return options;
+ }, {});
+
+ storage.configuration = {
+ ...storage.configuration,
+ ...validOptions,
+ };
+ }
+ return { ...meta, storage };
+}
+
+export function dataConnectorToFlattened(
+ dataConnector: DataConnectorRead | null
+): DataConnectorFlat {
+ if (!dataConnector) {
+ return EMPTY_DATA_CONNECTOR_FLAT;
+ }
+ const configurationOptions = dataConnector.storage.configuration
+ ? dataConnector.storage.configuration
+ : {};
+ const { type, provider, ...options } = configurationOptions; // eslint-disable-line @typescript-eslint/no-unused-vars
+ const flattened: DataConnectorFlat = {
+ dataConnectorId: dataConnector.id,
+ name: dataConnector.name,
+ namespace: dataConnector.namespace,
+ slug: dataConnector.slug,
+ visibility: dataConnector.visibility,
+ mountPoint: dataConnector.storage.target_path,
+ options,
+ provider: dataConnector.storage.configuration.provider
+ ? (dataConnector.storage.configuration.provider as string)
+ : undefined,
+ readOnly: dataConnector.storage.readonly,
+ schema: dataConnector.storage.configuration.type as string,
+ sourcePath: dataConnector.storage.source_path,
+ };
+
+ return flattened;
+}
+
+export function validationParametersFromDataConnectorConfiguration(
+ config: DataConnectorConfiguration
+) {
+ const newStorageDetails = dataConnectorToFlattened(
+ _dataConnectorFromConfig(config)
+ );
+ return _validationConfigurationFromDataConnectorFlat(newStorageDetails);
+}
+
+function _dataConnectorFromConfig(
+ config: DataConnectorConfiguration
+): DataConnectorRead {
+ const dataConnector = config.dataConnector;
+ const storageDefinition = config.dataConnector.storage;
+ const mergedDataConnector = { ...dataConnector, ...storageDefinition };
+ mergedDataConnector.configuration = { ...storageDefinition.configuration };
+ const sensitiveFieldValues = config.sensitiveFieldValues;
+ Object.entries(sensitiveFieldValues).forEach(([name, value]) => {
+ if (value != null && value !== "") {
+ mergedDataConnector.configuration[name] = value;
+ } else {
+ delete mergedDataConnector.configuration[name];
+ }
+ });
+ return mergedDataConnector;
+}
+
+function _validationConfigurationFromDataConnectorFlat(
+ dataConnector: DataConnectorFlat
+) {
+ const validateParameters: TestCloudStorageConnectionParams = {
+ configuration: {
+ type: dataConnector.schema,
+ },
+ source_path: dataConnector.sourcePath ?? "/",
+ };
+ if (dataConnector.provider) {
+ validateParameters.configuration.provider = dataConnector.provider;
+ }
+ if (dataConnector.options && Object.keys(dataConnector.options).length > 0) {
+ const options = dataConnector.options as CloudStorageDetailsOptions;
+ Object.entries(options).forEach(([key, value]) => {
+ if (value != undefined && value !== "") {
+ validateParameters.configuration[key] = value;
+ }
+ });
+ }
+
+ return validateParameters;
+}
diff --git a/client/src/features/dataConnectorsV2/components/useDataConnectorConfiguration.hook.ts b/client/src/features/dataConnectorsV2/components/useDataConnectorConfiguration.hook.ts
new file mode 100644
index 0000000000..ad30d866fc
--- /dev/null
+++ b/client/src/features/dataConnectorsV2/components/useDataConnectorConfiguration.hook.ts
@@ -0,0 +1,105 @@
+/*!
+ * Copyright 2024 - Swiss Data Science Center (SDSC)
+ * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+ * Eidgenössische Technische Hochschule Zürich (ETHZ).
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+
+import { useMemo } from "react";
+
+import { CLOUD_OPTIONS_OVERRIDE } from "../../project/components/cloudStorage/projectCloudStorage.constants";
+import { RCloneOption } from "../../projectsV2/api/data-connectors.api";
+import type {
+ DataConnectorRead,
+ CloudStorageSecretGet,
+} from "../../projectsV2/api/data-connectors.api";
+
+import type { SessionStartCloudStorageConfiguration } from "../../sessionsV2/startSessionOptionsV2.types";
+
+export interface DataConnectorConfiguration
+ extends Omit {
+ dataConnector: DataConnectorRead;
+}
+
+interface UseDataSourceConfigurationArgs {
+ dataConnectors: DataConnectorRead[] | undefined;
+}
+
+export default function useDataConnectorConfiguration({
+ dataConnectors,
+}: UseDataSourceConfigurationArgs) {
+ const dataConnectorConfigs = useMemo(
+ () =>
+ dataConnectors?.map((dataConnector) => {
+ const storageDefinition = dataConnector.storage;
+ const defSensitiveFieldsMap: Record = {};
+ if (dataConnector.storage.sensitive_fields != null) {
+ dataConnector.storage.sensitive_fields.forEach((f) => {
+ if (f.name != null) defSensitiveFieldsMap[f.name] = f;
+ });
+ }
+ const configSensitiveFields = Object.keys(
+ storageDefinition.configuration
+ ).filter((key) => defSensitiveFieldsMap[key] != null);
+
+ const overrides =
+ storageDefinition.storage_type != null
+ ? CLOUD_OPTIONS_OVERRIDE[storageDefinition.storage_type]
+ : undefined;
+
+ const sensitiveFieldDefinitions = configSensitiveFields
+ .filter((key) => defSensitiveFieldsMap[key].name != null)
+ .map((key) => {
+ const { help, name } = defSensitiveFieldsMap[key];
+ return {
+ help: overrides?.[key]?.help ?? help ?? "",
+ friendlyName: overrides?.[key]?.friendlyName ?? name ?? key,
+ name: name ?? key,
+ value: "",
+ };
+ });
+
+ const sensitiveFieldValues: SessionStartCloudStorageConfiguration["sensitiveFieldValues"] =
+ {};
+ configSensitiveFields.forEach((key) => {
+ const { name } = defSensitiveFieldsMap[key];
+ if (name == null) return;
+ sensitiveFieldValues[name] = "";
+ });
+ const storagesSecrets = dataConnectors?.reduce(
+ (a: Record, s) => {
+ a[s.id] = s.secrets ? s.secrets : [];
+ return a;
+ },
+ {}
+ );
+ const savedCredentialFields = storagesSecrets
+ ? storagesSecrets[dataConnector.id].map((s) => s.name)
+ : [];
+ return {
+ active: true,
+ dataConnector,
+ sensitiveFieldDefinitions,
+ sensitiveFieldValues,
+ saveCredentials: false,
+ savedCredentialFields,
+ };
+ }),
+ [dataConnectors]
+ );
+
+ return {
+ dataConnectorConfigs,
+ };
+}
diff --git a/client/src/features/groupsV2/show/GroupV2Show.tsx b/client/src/features/groupsV2/show/GroupV2Show.tsx
index 38cb300d6a..c5fac24e93 100644
--- a/client/src/features/groupsV2/show/GroupV2Show.tsx
+++ b/client/src/features/groupsV2/show/GroupV2Show.tsx
@@ -26,12 +26,14 @@ import {
useNavigate,
useParams,
} from "react-router-dom-v5-compat";
-import { Badge } from "reactstrap";
+import { Badge, Col, Row } from "reactstrap";
import { Loader } from "../../../components/Loader";
import ContainerWrap from "../../../components/container/ContainerWrap";
import LazyNotFound from "../../../not-found/LazyNotFound";
import { ABSOLUTE_ROUTES } from "../../../routing/routes.constants";
+
+import DataConnectorsBox from "../../dataConnectorsV2/components/DataConnectorsBox";
import MembershipGuard from "../../ProjectPageV2/utils/MembershipGuard";
import type { GroupResponse } from "../../projectsV2/api/namespace.api";
import {
@@ -134,6 +136,18 @@ export default function GroupV2Show() {
emptyListElement={No visible projects.
}
/>
+
+
);
}
diff --git a/client/src/features/project/components/cloudStorage/AddOrEditCloudStorage.tsx b/client/src/features/project/components/cloudStorage/AddOrEditCloudStorage.tsx
index f70e5071fd..ebd997d15b 100644
--- a/client/src/features/project/components/cloudStorage/AddOrEditCloudStorage.tsx
+++ b/client/src/features/project/components/cloudStorage/AddOrEditCloudStorage.tsx
@@ -162,7 +162,7 @@ export function AddStorageAdvancedToggle({
}
// *** Add storage: helpers *** //
-interface AddStorageStepProps {
+export interface AddStorageStepProps {
schema: CloudStorageSchema[];
setStorage: (newDetails: Partial) => void;
setState: (newState: Partial) => void;
diff --git a/client/src/features/project/utils/projectCloudStorage.utils.ts b/client/src/features/project/utils/projectCloudStorage.utils.ts
index d9fd91de2b..40c003fdbe 100644
--- a/client/src/features/project/utils/projectCloudStorage.utils.ts
+++ b/client/src/features/project/utils/projectCloudStorage.utils.ts
@@ -49,7 +49,16 @@ export interface CloudStorageOptions extends RCloneOption {
requiredCredential: boolean;
}
-type StorageDefinition = CloudStorage | CloudStorageGetRead;
+type SensitiveFields =
+ | CloudStorage["sensitive_fields"]
+ | CloudStorageGetRead["sensitive_fields"];
+type StorageConfiguration =
+ | CloudStorage["storage"]["configuration"]
+ | CloudStorageGetRead["storage"]["configuration"];
+type StorageAndSensitiveFieldsDefinition = {
+ storage: { configuration: StorageConfiguration };
+ sensitive_fields?: SensitiveFields;
+};
export function parseCloudStorageConfiguration(
formattedConfiguration: string
@@ -92,7 +101,9 @@ export function convertFromAdvancedConfig(
return values.length ? values.join("\n") + "\n" : "";
}
-export function getCredentialFieldDefinitions(
+export function getCredentialFieldDefinitions<
+ T extends StorageAndSensitiveFieldsDefinition
+>(
storageDefinition: T
):
| (T extends CloudStorageGetRead
diff --git a/client/src/features/projectsV2/api/data-connectors.api-config.ts b/client/src/features/projectsV2/api/data-connectors.api-config.ts
new file mode 100644
index 0000000000..91bdbe184f
--- /dev/null
+++ b/client/src/features/projectsV2/api/data-connectors.api-config.ts
@@ -0,0 +1,33 @@
+/*!
+ * Copyright 2024 - Swiss Data Science Center (SDSC)
+ * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+ * Eidgenössische Technische Hochschule Zürich (ETHZ).
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Run `npm run generate-api:namespaceV2` to generate the API
+import type { ConfigFile } from "@rtk-query/codegen-openapi";
+import path from "path";
+
+const config: ConfigFile = {
+ // Configure to inject endpoints into the dataConnectorsApi
+ apiFile: "./data-connectors.empty-api.ts",
+ apiImport: "dataConnectorsEmptyApi",
+ outputFile: "./data-connectors.api.ts",
+ exportName: "dataConnectorsApi",
+ hooks: true,
+ schemaFile: path.join(__dirname, "data-connectors.openapi.json"),
+};
+
+export default config;
diff --git a/client/src/features/projectsV2/api/data-connectors.api.ts b/client/src/features/projectsV2/api/data-connectors.api.ts
new file mode 100644
index 0000000000..b52b7353d6
--- /dev/null
+++ b/client/src/features/projectsV2/api/data-connectors.api.ts
@@ -0,0 +1,354 @@
+import { dataConnectorsEmptyApi as api } from "./data-connectors.empty-api";
+const injectedRtkApi = api.injectEndpoints({
+ endpoints: (build) => ({
+ getDataConnectors: build.query<
+ GetDataConnectorsApiResponse,
+ GetDataConnectorsApiArg
+ >({
+ query: (queryArg) => ({
+ url: `/data_connectors`,
+ params: { params: queryArg.params },
+ }),
+ }),
+ postDataConnectors: build.mutation<
+ PostDataConnectorsApiResponse,
+ PostDataConnectorsApiArg
+ >({
+ query: (queryArg) => ({
+ url: `/data_connectors`,
+ method: "POST",
+ body: queryArg.dataConnectorPost,
+ }),
+ }),
+ getDataConnectorsByDataConnectorId: build.query<
+ GetDataConnectorsByDataConnectorIdApiResponse,
+ GetDataConnectorsByDataConnectorIdApiArg
+ >({
+ query: (queryArg) => ({
+ url: `/data_connectors/${queryArg.dataConnectorId}`,
+ }),
+ }),
+ patchDataConnectorsByDataConnectorId: build.mutation<
+ PatchDataConnectorsByDataConnectorIdApiResponse,
+ PatchDataConnectorsByDataConnectorIdApiArg
+ >({
+ query: (queryArg) => ({
+ url: `/data_connectors/${queryArg.dataConnectorId}`,
+ method: "PATCH",
+ body: queryArg.dataConnectorPatch,
+ headers: { "If-Match": queryArg["If-Match"] },
+ }),
+ }),
+ deleteDataConnectorsByDataConnectorId: build.mutation<
+ DeleteDataConnectorsByDataConnectorIdApiResponse,
+ DeleteDataConnectorsByDataConnectorIdApiArg
+ >({
+ query: (queryArg) => ({
+ url: `/data_connectors/${queryArg.dataConnectorId}`,
+ method: "DELETE",
+ }),
+ }),
+ getNamespacesByNamespaceDataConnectorsAndSlug: build.query<
+ GetNamespacesByNamespaceDataConnectorsAndSlugApiResponse,
+ GetNamespacesByNamespaceDataConnectorsAndSlugApiArg
+ >({
+ query: (queryArg) => ({
+ url: `/namespaces/${queryArg["namespace"]}/data_connectors/${queryArg.slug}`,
+ }),
+ }),
+ getDataConnectorsByDataConnectorIdSecrets: build.query<
+ GetDataConnectorsByDataConnectorIdSecretsApiResponse,
+ GetDataConnectorsByDataConnectorIdSecretsApiArg
+ >({
+ query: (queryArg) => ({
+ url: `/data_connectors/${queryArg.dataConnectorId}/secrets`,
+ }),
+ }),
+ postDataConnectorsByDataConnectorIdSecrets: build.mutation<
+ PostDataConnectorsByDataConnectorIdSecretsApiResponse,
+ PostDataConnectorsByDataConnectorIdSecretsApiArg
+ >({
+ query: (queryArg) => ({
+ url: `/data_connectors/${queryArg.dataConnectorId}/secrets`,
+ method: "POST",
+ body: queryArg.cloudStorageSecretPostList,
+ }),
+ }),
+ deleteDataConnectorsByDataConnectorIdSecrets: build.mutation<
+ DeleteDataConnectorsByDataConnectorIdSecretsApiResponse,
+ DeleteDataConnectorsByDataConnectorIdSecretsApiArg
+ >({
+ query: (queryArg) => ({
+ url: `/data_connectors/${queryArg.dataConnectorId}/secrets`,
+ method: "DELETE",
+ }),
+ }),
+ }),
+ overrideExisting: false,
+});
+export { injectedRtkApi as dataConnectorsApi };
+export type GetDataConnectorsApiResponse =
+ /** status 200 List of data connectors */ DataConnectorsListRead;
+export type GetDataConnectorsApiArg = {
+ /** query parameters */
+ params?: DataConnectorsGetQuery;
+};
+export type PostDataConnectorsApiResponse =
+ /** status 201 The data connector was created */ DataConnectorRead;
+export type PostDataConnectorsApiArg = {
+ dataConnectorPost: DataConnectorPost;
+};
+export type GetDataConnectorsByDataConnectorIdApiResponse =
+ /** status 200 The data connector */ DataConnectorRead;
+export type GetDataConnectorsByDataConnectorIdApiArg = {
+ /** the ID of the data connector */
+ dataConnectorId: Ulid;
+};
+export type PatchDataConnectorsByDataConnectorIdApiResponse =
+ /** status 200 The patched data connector */ DataConnectorRead;
+export type PatchDataConnectorsByDataConnectorIdApiArg = {
+ /** the ID of the data connector */
+ dataConnectorId: Ulid;
+ /** If-Match header, for avoiding mid-air collisions */
+ "If-Match": ETag;
+ dataConnectorPatch: DataConnectorPatch;
+};
+export type DeleteDataConnectorsByDataConnectorIdApiResponse =
+ /** status 204 The data connector was removed or did not exist in the first place */ void;
+export type DeleteDataConnectorsByDataConnectorIdApiArg = {
+ /** the ID of the data connector */
+ dataConnectorId: Ulid;
+};
+export type GetNamespacesByNamespaceDataConnectorsAndSlugApiResponse =
+ /** status 200 The data connectors */ DataConnectorRead;
+export type GetNamespacesByNamespaceDataConnectorsAndSlugApiArg = {
+ namespace: string;
+ slug: string;
+};
+export type GetDataConnectorsByDataConnectorIdSecretsApiResponse =
+ /** status 200 The saved storage secrets */ CloudStorageSecretGetList;
+export type GetDataConnectorsByDataConnectorIdSecretsApiArg = {
+ /** the ID of the data connector */
+ dataConnectorId: Ulid;
+};
+export type PostDataConnectorsByDataConnectorIdSecretsApiResponse =
+ /** status 201 The secrets for cloud storage were saved */ CloudStorageSecretGetList;
+export type PostDataConnectorsByDataConnectorIdSecretsApiArg = {
+ /** the ID of the data connector */
+ dataConnectorId: Ulid;
+ cloudStorageSecretPostList: CloudStorageSecretPostList;
+};
+export type DeleteDataConnectorsByDataConnectorIdSecretsApiResponse =
+ /** status 204 The secrets were removed or did not exist in the first place or the storage doesn't exist */ void;
+export type DeleteDataConnectorsByDataConnectorIdSecretsApiArg = {
+ /** the ID of the data connector */
+ dataConnectorId: Ulid;
+};
+export type Ulid = string;
+export type DataConnectorName = string;
+export type Slug = string;
+export type StorageType = string;
+export type StorageTypeRead = string;
+export type RCloneConfig = {
+ [key: string]: number | (string | null) | boolean | object;
+};
+export type SourcePath = string;
+export type TargetPath = string;
+export type StorageReadOnly = boolean;
+export type RCloneOption = {
+ /** name of the option */
+ name?: string;
+ /** help text for the option */
+ help?: string;
+ /** The cloud provider the option is for (See 'provider' RCloneOption in the schema for potential values) */
+ provider?: string;
+ /** default value for the option */
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ default?: number | string | boolean | object | any;
+ /** string representation of the default value */
+ default_str?: string;
+ /** These list potential values for this option, like an enum. With `exclusive: true`, only a value from the list is allowed. */
+ examples?: {
+ /** a potential value for the option (think enum) */
+ value?: string;
+ /** help text for the value */
+ help?: string;
+ /** The provider this value is applicable for. Empty if valid for all providers. */
+ provider?: string;
+ }[];
+ /** whether the option is required or not */
+ required?: boolean;
+ /** whether the field is a password (use **** for display) */
+ ispassword?: boolean;
+ /** whether the value is sensitive (not stored in the service). Do not send this in requests to the service. */
+ sensitive?: boolean;
+ /** whether this is an advanced config option (probably don't show these to users) */
+ advanced?: boolean;
+ /** if true, only values from 'examples' can be used */
+ exclusive?: boolean;
+ /** data type of option value. RClone has more options but they map to the ones listed here. */
+ datatype?: "int" | "bool" | "string" | "Time";
+};
+export type CloudStorageCore = {
+ storage_type: StorageType;
+ configuration: RCloneConfig;
+ source_path: SourcePath;
+ target_path: TargetPath;
+ readonly: StorageReadOnly;
+ sensitive_fields: RCloneOption[];
+};
+export type CloudStorageCoreRead = {
+ storage_type: StorageTypeRead;
+ configuration: RCloneConfig;
+ source_path: SourcePath;
+ target_path: TargetPath;
+ readonly: StorageReadOnly;
+ sensitive_fields: RCloneOption[];
+};
+export type CloudStorageSecretGet = {
+ /** Name of the field to store credential for */
+ name: string;
+ secret_id: Ulid;
+};
+export type CreationDate = string;
+export type UserId = string;
+export type Visibility = "private" | "public";
+export type Description = string;
+export type ETag = string;
+export type Keyword = string;
+export type KeywordsList = Keyword[];
+export type DataConnector = {
+ id: Ulid;
+ name: DataConnectorName;
+ namespace: Slug;
+ slug: Slug;
+ storage: CloudStorageCore;
+ secrets?: CloudStorageSecretGet[];
+ creation_date: CreationDate;
+ created_by: UserId;
+ visibility: Visibility;
+ description?: Description;
+ etag: ETag;
+ keywords?: KeywordsList;
+};
+export type DataConnectorRead = {
+ id: Ulid;
+ name: DataConnectorName;
+ namespace: Slug;
+ slug: Slug;
+ storage: CloudStorageCoreRead;
+ secrets?: CloudStorageSecretGet[];
+ creation_date: CreationDate;
+ created_by: UserId;
+ visibility: Visibility;
+ description?: Description;
+ etag: ETag;
+ keywords?: KeywordsList;
+};
+export type DataConnectorsList = DataConnector[];
+export type DataConnectorsListRead = DataConnectorRead[];
+export type ErrorResponse = {
+ error: {
+ code: number;
+ detail?: string;
+ message: string;
+ };
+};
+export type PaginationRequest = {
+ /** Result's page number starting from 1 */
+ page?: number;
+ /** The number of results per page */
+ per_page?: number;
+};
+export type DataConnectorsGetQuery = PaginationRequest & {
+ /** A namespace, used as a filter. */
+ namespace?: string;
+};
+export type CloudStorageCorePost = {
+ storage_type?: StorageType;
+ configuration: RCloneConfig;
+ source_path: SourcePath;
+ target_path: TargetPath;
+ readonly?: StorageReadOnly;
+};
+export type CloudStorageCorePostRead = {
+ storage_type?: StorageTypeRead;
+ configuration: RCloneConfig;
+ source_path: SourcePath;
+ target_path: TargetPath;
+ readonly?: StorageReadOnly;
+};
+export type CloudStorageUrlV2 = {
+ storage_url: string;
+ target_path: TargetPath;
+ readonly?: StorageReadOnly;
+};
+export type DataConnectorPost = {
+ name: DataConnectorName;
+ namespace: Slug;
+ slug?: Slug;
+ storage: CloudStorageCorePost | CloudStorageUrlV2;
+ visibility?: Visibility;
+ description?: Description;
+ keywords?: KeywordsList;
+};
+export type DataConnectorPostRead = {
+ name: DataConnectorName;
+ namespace: Slug;
+ slug?: Slug;
+ storage: CloudStorageCorePostRead | CloudStorageUrlV2;
+ visibility?: Visibility;
+ description?: Description;
+ keywords?: KeywordsList;
+};
+export type CloudStorageCorePatch = {
+ storage_type?: StorageType;
+ configuration?: RCloneConfig;
+ source_path?: SourcePath;
+ target_path?: TargetPath;
+ readonly?: StorageReadOnly;
+};
+export type CloudStorageCorePatchRead = {
+ storage_type?: StorageTypeRead;
+ configuration?: RCloneConfig;
+ source_path?: SourcePath;
+ target_path?: TargetPath;
+ readonly?: StorageReadOnly;
+};
+export type DataConnectorPatch = {
+ name?: DataConnectorName;
+ namespace?: Slug;
+ slug?: Slug;
+ storage?: CloudStorageCorePatch;
+ visibility?: Visibility;
+ description?: Description;
+ keywords?: KeywordsList;
+};
+export type DataConnectorPatchRead = {
+ name?: DataConnectorName;
+ namespace?: Slug;
+ slug?: Slug;
+ storage?: CloudStorageCorePatchRead;
+ visibility?: Visibility;
+ description?: Description;
+ keywords?: KeywordsList;
+};
+export type CloudStorageSecretGetList = CloudStorageSecretGet[];
+export type SecretValue = string;
+export type CloudStorageSecretPost = {
+ /** Name of the field to store credential for */
+ name: string;
+ value: SecretValue;
+};
+export type CloudStorageSecretPostList = CloudStorageSecretPost[];
+export const {
+ useGetDataConnectorsQuery,
+ usePostDataConnectorsMutation,
+ useGetDataConnectorsByDataConnectorIdQuery,
+ usePatchDataConnectorsByDataConnectorIdMutation,
+ useDeleteDataConnectorsByDataConnectorIdMutation,
+ useGetNamespacesByNamespaceDataConnectorsAndSlugQuery,
+ useGetDataConnectorsByDataConnectorIdSecretsQuery,
+ usePostDataConnectorsByDataConnectorIdSecretsMutation,
+ useDeleteDataConnectorsByDataConnectorIdSecretsMutation,
+} = injectedRtkApi;
diff --git a/client/src/features/projectsV2/api/data-connectors.empty-api.ts b/client/src/features/projectsV2/api/data-connectors.empty-api.ts
new file mode 100644
index 0000000000..c8bdefe3f1
--- /dev/null
+++ b/client/src/features/projectsV2/api/data-connectors.empty-api.ts
@@ -0,0 +1,26 @@
+/*!
+ * Copyright 2023 - Swiss Data Science Center (SDSC)
+ * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+ * Eidgenössische Technische Hochschule Zürich (ETHZ).
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { createApi, fetchBaseQuery } from "@reduxjs/toolkit/dist/query/react";
+
+// initialize an empty api service that we'll inject endpoints into later as needed
+export const dataConnectorsEmptyApi = createApi({
+ baseQuery: fetchBaseQuery({ baseUrl: "/ui-server/api/data" }),
+ endpoints: () => ({}),
+ reducerPath: "dataConnectorsApi",
+});
diff --git a/client/src/features/projectsV2/api/data-connectors.enhanced-api.ts b/client/src/features/projectsV2/api/data-connectors.enhanced-api.ts
new file mode 100644
index 0000000000..90ca0ae5a8
--- /dev/null
+++ b/client/src/features/projectsV2/api/data-connectors.enhanced-api.ts
@@ -0,0 +1,86 @@
+import { AbstractKgPaginatedResponse } from "../../../utils/types/pagination.types";
+import { processPaginationHeaders } from "../../../utils/helpers/kgPagination.utils";
+
+import { dataConnectorsApi as api } from "./data-connectors.api";
+
+import type {
+ GetDataConnectorsApiArg,
+ GetDataConnectorsApiResponse as GetDataConnectorsApiResponseOrig,
+} from "./data-connectors.api";
+
+export interface GetDataConnectorsApiResponse
+ extends AbstractKgPaginatedResponse {
+ dataConnectors: GetDataConnectorsApiResponseOrig;
+}
+
+const injectedApi = api.injectEndpoints({
+ endpoints: (builder) => ({
+ getDataConnectorsPaged: builder.query<
+ GetDataConnectorsApiResponse,
+ GetDataConnectorsApiArg
+ >({
+ query: (queryArg) => ({
+ url: "/data_connectors",
+ params: {
+ namespace: queryArg.params?.namespace,
+ page: queryArg.params?.page,
+ per_page: queryArg.params?.per_page,
+ },
+ }),
+ transformResponse: (response, meta, queryArg) => {
+ const dataConnectors = response as GetDataConnectorsApiResponseOrig;
+ const headers = meta?.response?.headers;
+ const headerResponse = processPaginationHeaders(
+ headers,
+ queryArg.params == null
+ ? {}
+ : { page: queryArg.params.page, perPage: queryArg.params.per_page },
+ dataConnectors
+ );
+
+ return {
+ dataConnectors,
+ page: headerResponse.page,
+ perPage: headerResponse.perPage,
+ total: headerResponse.total,
+ totalPages: headerResponse.totalPages,
+ };
+ },
+ }),
+ }),
+});
+
+const enhancedApi = injectedApi.enhanceEndpoints({
+ addTagTypes: ["DataConnectors", "DataConnectorSecrets"],
+ endpoints: {
+ deleteDataConnectorsByDataConnectorId: {
+ invalidatesTags: ["DataConnectors"],
+ },
+ deleteDataConnectorsByDataConnectorIdSecrets: {
+ invalidatesTags: ["DataConnectorSecrets"],
+ },
+ getDataConnectorsPaged: {
+ providesTags: ["DataConnectors"],
+ },
+ patchDataConnectorsByDataConnectorId: {
+ invalidatesTags: ["DataConnectors"],
+ },
+ postDataConnectors: {
+ invalidatesTags: ["DataConnectors"],
+ },
+ postDataConnectorsByDataConnectorIdSecrets: {
+ invalidatesTags: ["DataConnectorSecrets"],
+ },
+ },
+});
+
+export { enhancedApi as dataConnectorsApi };
+export const {
+ // data connectors hooks
+ useDeleteDataConnectorsByDataConnectorIdMutation,
+ useDeleteDataConnectorsByDataConnectorIdSecretsMutation,
+ useGetDataConnectorsPagedQuery: useGetDataConnectorsQuery,
+ usePatchDataConnectorsByDataConnectorIdMutation,
+ usePostDataConnectorsMutation,
+ usePostDataConnectorsByDataConnectorIdSecretsMutation,
+} = enhancedApi;
diff --git a/client/src/features/projectsV2/api/data-connectors.openapi.json b/client/src/features/projectsV2/api/data-connectors.openapi.json
new file mode 100644
index 0000000000..2366677f89
--- /dev/null
+++ b/client/src/features/projectsV2/api/data-connectors.openapi.json
@@ -0,0 +1,940 @@
+{
+ "openapi": "3.0.2",
+ "info": {
+ "title": "Renku Data Services API",
+ "description": "This service is the main backend for Renku. It provides information about users, projects,\ncloud storage, access to compute resources and many other things.\n",
+ "version": "v1"
+ },
+ "servers": [
+ {
+ "url": "/api/data"
+ },
+ {
+ "url": "/ui-server/api/data"
+ }
+ ],
+ "paths": {
+ "/data_connectors": {
+ "get": {
+ "summary": "Get all data connectors",
+ "parameters": [
+ {
+ "in": "query",
+ "description": "query parameters",
+ "name": "params",
+ "style": "form",
+ "explode": true,
+ "schema": {
+ "$ref": "#/components/schemas/DataConnectorsGetQuery"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "List of data connectors",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/DataConnectorsList"
+ }
+ }
+ },
+ "headers": {
+ "page": {
+ "description": "The index of the current page (starting at 1).",
+ "required": true,
+ "schema": {
+ "type": "integer"
+ }
+ },
+ "per-page": {
+ "description": "The number of items per page.",
+ "required": true,
+ "schema": {
+ "type": "integer"
+ }
+ },
+ "total": {
+ "description": "The total number of items.",
+ "required": true,
+ "schema": {
+ "type": "integer"
+ }
+ },
+ "total-pages": {
+ "description": "The total number of pages.",
+ "required": true,
+ "schema": {
+ "type": "integer"
+ }
+ }
+ }
+ },
+ "default": {
+ "$ref": "#/components/responses/Error"
+ }
+ },
+ "tags": ["data connectors"]
+ },
+ "post": {
+ "summary": "Create a new data connector",
+ "requestBody": {
+ "required": true,
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/DataConnectorPost"
+ }
+ }
+ }
+ },
+ "responses": {
+ "201": {
+ "description": "The data connector was created",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/DataConnector"
+ }
+ }
+ }
+ },
+ "default": {
+ "$ref": "#/components/responses/Error"
+ }
+ },
+ "tags": ["data connectors"]
+ }
+ },
+ "/data_connectors/{data_connector_id}": {
+ "parameters": [
+ {
+ "in": "path",
+ "name": "data_connector_id",
+ "required": true,
+ "schema": {
+ "$ref": "#/components/schemas/Ulid"
+ },
+ "description": "the ID of the data connector"
+ }
+ ],
+ "get": {
+ "summary": "Get data connector details",
+ "responses": {
+ "200": {
+ "description": "The data connector",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/DataConnector"
+ }
+ }
+ }
+ },
+ "404": {
+ "description": "The data connector does not exist",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "default": {
+ "$ref": "#/components/responses/Error"
+ }
+ },
+ "tags": ["data connectors"]
+ },
+ "patch": {
+ "summary": "Update specific fields of an existing data connector",
+ "parameters": [
+ {
+ "$ref": "#/components/parameters/If-Match"
+ }
+ ],
+ "requestBody": {
+ "required": true,
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/DataConnectorPatch"
+ }
+ }
+ }
+ },
+ "responses": {
+ "200": {
+ "description": "The patched data connector",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/DataConnector"
+ }
+ }
+ }
+ },
+ "404": {
+ "description": "The data connector does not exist",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "default": {
+ "$ref": "#/components/responses/Error"
+ }
+ },
+ "tags": ["data connectors"]
+ },
+ "delete": {
+ "summary": "Remove a data connector",
+ "responses": {
+ "204": {
+ "description": "The data connector was removed or did not exist in the first place"
+ },
+ "default": {
+ "$ref": "#/components/responses/Error"
+ }
+ },
+ "tags": ["data connectors"]
+ }
+ },
+ "/namespaces/{namespace}/data_connectors/{slug}": {
+ "parameters": [
+ {
+ "in": "path",
+ "name": "namespace",
+ "required": true,
+ "schema": {
+ "type": "string"
+ }
+ },
+ {
+ "in": "path",
+ "name": "slug",
+ "required": true,
+ "schema": {
+ "type": "string"
+ }
+ }
+ ],
+ "get": {
+ "summary": "Get a data connector by namespace and project slug",
+ "responses": {
+ "200": {
+ "description": "The data connectors",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/DataConnector"
+ }
+ }
+ }
+ },
+ "404": {
+ "description": "The data connector does not exist",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "default": {
+ "$ref": "#/components/responses/Error"
+ }
+ },
+ "tags": ["data connectors"]
+ }
+ },
+ "/data_connectors/{data_connector_id}/secrets": {
+ "parameters": [
+ {
+ "in": "path",
+ "name": "data_connector_id",
+ "required": true,
+ "schema": {
+ "$ref": "#/components/schemas/Ulid"
+ },
+ "description": "the ID of the data connector"
+ }
+ ],
+ "get": {
+ "summary": "Get all saved secrets for a data connector",
+ "responses": {
+ "200": {
+ "description": "The saved storage secrets",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/CloudStorageSecretGetList"
+ }
+ }
+ }
+ },
+ "404": {
+ "description": "Storage was not found",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "default": {
+ "$ref": "#/components/responses/Error"
+ }
+ },
+ "tags": ["data connectors"]
+ },
+ "post": {
+ "summary": "Save secrets for a data connector",
+ "requestBody": {
+ "required": true,
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/CloudStorageSecretPostList"
+ }
+ }
+ }
+ },
+ "responses": {
+ "201": {
+ "description": "The secrets for cloud storage were saved",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/CloudStorageSecretGetList"
+ }
+ }
+ }
+ },
+ "default": {
+ "$ref": "#/components/responses/Error"
+ }
+ },
+ "tags": ["data connectors"]
+ },
+ "delete": {
+ "summary": "Remove all saved secrets for a data connector",
+ "responses": {
+ "204": {
+ "description": "The secrets were removed or did not exist in the first place or the storage doesn't exist"
+ },
+ "default": {
+ "$ref": "#/components/responses/Error"
+ }
+ },
+ "tags": ["data connectors"]
+ }
+ }
+ },
+ "components": {
+ "schemas": {
+ "DataConnectorsList": {
+ "description": "A list of data connectors",
+ "type": "array",
+ "items": {
+ "$ref": "#/components/schemas/DataConnector"
+ }
+ },
+ "DataConnector": {
+ "description": "A data connector for Renku 2.0 for mounting remote data storage\n",
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "id": {
+ "$ref": "#/components/schemas/Ulid"
+ },
+ "name": {
+ "$ref": "#/components/schemas/DataConnectorName"
+ },
+ "namespace": {
+ "$ref": "#/components/schemas/Slug"
+ },
+ "slug": {
+ "$ref": "#/components/schemas/Slug"
+ },
+ "storage": {
+ "$ref": "#/components/schemas/CloudStorageCore"
+ },
+ "secrets": {
+ "type": "array",
+ "items": {
+ "$ref": "#/components/schemas/CloudStorageSecretGet"
+ }
+ },
+ "creation_date": {
+ "$ref": "#/components/schemas/CreationDate"
+ },
+ "created_by": {
+ "$ref": "#/components/schemas/UserId"
+ },
+ "visibility": {
+ "$ref": "#/components/schemas/Visibility"
+ },
+ "description": {
+ "$ref": "#/components/schemas/Description"
+ },
+ "etag": {
+ "$ref": "#/components/schemas/ETag"
+ },
+ "keywords": {
+ "$ref": "#/components/schemas/KeywordsList"
+ }
+ },
+ "required": [
+ "id",
+ "name",
+ "namespace",
+ "slug",
+ "storage",
+ "creation_date",
+ "created_by",
+ "visibility",
+ "etag"
+ ]
+ },
+ "DataConnectorPost": {
+ "description": "A data connector to be created in Renku 2.0\n",
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "name": {
+ "$ref": "#/components/schemas/DataConnectorName"
+ },
+ "namespace": {
+ "$ref": "#/components/schemas/Slug"
+ },
+ "slug": {
+ "$ref": "#/components/schemas/Slug"
+ },
+ "storage": {
+ "oneOf": [
+ {
+ "$ref": "#/components/schemas/CloudStorageCorePost"
+ },
+ {
+ "$ref": "#/components/schemas/CloudStorageUrlV2"
+ }
+ ]
+ },
+ "visibility": {
+ "$ref": "#/components/schemas/Visibility",
+ "default": "private"
+ },
+ "description": {
+ "$ref": "#/components/schemas/Description"
+ },
+ "keywords": {
+ "$ref": "#/components/schemas/KeywordsList"
+ }
+ },
+ "required": ["name", "namespace", "storage"]
+ },
+ "DataConnectorPatch": {
+ "description": "Patch of a data connector\n",
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "name": {
+ "$ref": "#/components/schemas/DataConnectorName"
+ },
+ "namespace": {
+ "$ref": "#/components/schemas/Slug"
+ },
+ "slug": {
+ "$ref": "#/components/schemas/Slug"
+ },
+ "storage": {
+ "$ref": "#/components/schemas/CloudStorageCorePatch"
+ },
+ "visibility": {
+ "$ref": "#/components/schemas/Visibility"
+ },
+ "description": {
+ "$ref": "#/components/schemas/Description"
+ },
+ "keywords": {
+ "$ref": "#/components/schemas/KeywordsList"
+ }
+ }
+ },
+ "CloudStorageCore": {
+ "description": "Represents the configuration used to mount remote data storage",
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "storage_type": {
+ "$ref": "#/components/schemas/StorageType"
+ },
+ "configuration": {
+ "$ref": "#/components/schemas/RCloneConfig"
+ },
+ "source_path": {
+ "$ref": "#/components/schemas/SourcePath"
+ },
+ "target_path": {
+ "$ref": "#/components/schemas/TargetPath"
+ },
+ "readonly": {
+ "$ref": "#/components/schemas/StorageReadOnly"
+ },
+ "sensitive_fields": {
+ "type": "array",
+ "items": {
+ "$ref": "#/components/schemas/RCloneOption"
+ }
+ }
+ },
+ "required": [
+ "storage_type",
+ "configuration",
+ "source_path",
+ "target_path",
+ "readonly",
+ "sensitive_fields"
+ ]
+ },
+ "CloudStorageCorePost": {
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "storage_type": {
+ "$ref": "#/components/schemas/StorageType"
+ },
+ "configuration": {
+ "$ref": "#/components/schemas/RCloneConfig"
+ },
+ "source_path": {
+ "$ref": "#/components/schemas/SourcePath"
+ },
+ "target_path": {
+ "$ref": "#/components/schemas/TargetPath"
+ },
+ "readonly": {
+ "$ref": "#/components/schemas/StorageReadOnly",
+ "default": true
+ }
+ },
+ "required": ["configuration", "source_path", "target_path"]
+ },
+ "CloudStorageCorePatch": {
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "storage_type": {
+ "$ref": "#/components/schemas/StorageType"
+ },
+ "configuration": {
+ "$ref": "#/components/schemas/RCloneConfig"
+ },
+ "source_path": {
+ "$ref": "#/components/schemas/SourcePath"
+ },
+ "target_path": {
+ "$ref": "#/components/schemas/TargetPath"
+ },
+ "readonly": {
+ "$ref": "#/components/schemas/StorageReadOnly"
+ }
+ }
+ },
+ "RCloneConfig": {
+ "type": "object",
+ "description": "Dictionary of rclone key:value pairs (based on schema from '/storage_schema')",
+ "additionalProperties": {
+ "oneOf": [
+ {
+ "type": "integer"
+ },
+ {
+ "type": "string",
+ "nullable": true
+ },
+ {
+ "type": "boolean"
+ },
+ {
+ "type": "object"
+ }
+ ]
+ }
+ },
+ "CloudStorageUrlV2": {
+ "type": "object",
+ "properties": {
+ "storage_url": {
+ "type": "string"
+ },
+ "target_path": {
+ "$ref": "#/components/schemas/TargetPath"
+ },
+ "readonly": {
+ "$ref": "#/components/schemas/StorageReadOnly",
+ "default": true
+ }
+ },
+ "required": ["storage_url", "target_path"],
+ "example": {
+ "storage_url": "s3://giab"
+ }
+ },
+ "CloudStorageSecretPost": {
+ "type": "object",
+ "description": "Data for storing secret for a storage field",
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "Name of the field to store credential for",
+ "minLength": 1,
+ "maxLength": 99
+ },
+ "value": {
+ "$ref": "#/components/schemas/SecretValue"
+ }
+ },
+ "required": ["name", "value"]
+ },
+ "CloudStorageSecretPostList": {
+ "description": "List of storage secrets that are saved",
+ "type": "array",
+ "items": {
+ "$ref": "#/components/schemas/CloudStorageSecretPost"
+ }
+ },
+ "CloudStorageSecretGetList": {
+ "description": "List of storage secrets that are saved",
+ "type": "array",
+ "items": {
+ "$ref": "#/components/schemas/CloudStorageSecretGet"
+ }
+ },
+ "CloudStorageSecretGet": {
+ "type": "object",
+ "description": "Data for saved storage secrets",
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "Name of the field to store credential for",
+ "minLength": 1,
+ "maxLength": 99
+ },
+ "secret_id": {
+ "$ref": "#/components/schemas/Ulid"
+ }
+ },
+ "required": ["name", "secret_id"]
+ },
+ "SecretValue": {
+ "description": "Secret value that can be any text",
+ "type": "string",
+ "minLength": 1,
+ "maxLength": 5000
+ },
+ "RCloneEntry": {
+ "type": "object",
+ "description": "Schema for a storage type in rclone, like S3 or Azure Blob Storage. Contains fields for that storage type.",
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "Human readable name of the provider"
+ },
+ "description": {
+ "type": "string",
+ "description": "description of the provider"
+ },
+ "prefix": {
+ "type": "string",
+ "description": "Machine readable name of the provider"
+ },
+ "options": {
+ "description": "Fields/properties used for this storage.",
+ "type": "array",
+ "items": {
+ "$ref": "#/components/schemas/RCloneOption"
+ }
+ }
+ }
+ },
+ "RCloneOption": {
+ "type": "object",
+ "description": "Single field on an RClone storage, like \"remote\" or \"access_key_id\"",
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "name of the option"
+ },
+ "help": {
+ "type": "string",
+ "description": "help text for the option"
+ },
+ "provider": {
+ "type": "string",
+ "description": "The cloud provider the option is for (See 'provider' RCloneOption in the schema for potential values)",
+ "example": "AWS"
+ },
+ "default": {
+ "oneOf": [
+ {
+ "type": "number"
+ },
+ {
+ "type": "string"
+ },
+ {
+ "type": "boolean"
+ },
+ {
+ "type": "object"
+ },
+ {
+ "type": "array"
+ }
+ ],
+ "description": "default value for the option"
+ },
+ "default_str": {
+ "type": "string",
+ "description": "string representation of the default value"
+ },
+ "examples": {
+ "description": "These list potential values for this option, like an enum. With `exclusive: true`, only a value from the list is allowed.",
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "value": {
+ "type": "string",
+ "description": "a potential value for the option (think enum)"
+ },
+ "help": {
+ "type": "string",
+ "description": "help text for the value"
+ },
+ "provider": {
+ "type": "string",
+ "description": "The provider this value is applicable for. Empty if valid for all providers."
+ }
+ }
+ }
+ },
+ "required": {
+ "type": "boolean",
+ "description": "whether the option is required or not"
+ },
+ "ispassword": {
+ "type": "boolean",
+ "description": "whether the field is a password (use **** for display)"
+ },
+ "sensitive": {
+ "type": "boolean",
+ "description": "whether the value is sensitive (not stored in the service). Do not send this in requests to the service."
+ },
+ "advanced": {
+ "type": "boolean",
+ "description": "whether this is an advanced config option (probably don't show these to users)"
+ },
+ "exclusive": {
+ "type": "boolean",
+ "description": "if true, only values from 'examples' can be used"
+ },
+ "datatype": {
+ "type": "string",
+ "description": "data type of option value. RClone has more options but they map to the ones listed here.",
+ "enum": ["int", "bool", "string", "Time"]
+ }
+ }
+ },
+ "Ulid": {
+ "description": "ULID identifier",
+ "type": "string",
+ "minLength": 26,
+ "maxLength": 26,
+ "pattern": "^[0-7][0-9A-HJKMNP-TV-Z]{25}$"
+ },
+ "Slug": {
+ "description": "A command-line/url friendly name for a namespace",
+ "type": "string",
+ "minLength": 1,
+ "maxLength": 99,
+ "pattern": "^(?!.*\\.git$|.*\\.atom$|.*[\\-._][\\-._].*)[a-zA-Z0-9][a-zA-Z0-9\\-_.]*$",
+ "example": "a-slug-example"
+ },
+ "CreationDate": {
+ "description": "The date and time the resource was created (in UTC and ISO-8601 format)",
+ "type": "string",
+ "format": "date-time",
+ "example": "2023-11-01T17:32:28Z"
+ },
+ "UserId": {
+ "type": "string",
+ "description": "Keycloak user ID",
+ "example": "f74a228b-1790-4276-af5f-25c2424e9b0c",
+ "pattern": "^[A-Za-z0-9]{1}[A-Za-z0-9-]+$"
+ },
+ "Visibility": {
+ "description": "Project's visibility levels",
+ "type": "string",
+ "enum": ["private", "public"]
+ },
+ "Description": {
+ "description": "A description for the resource",
+ "type": "string",
+ "maxLength": 500
+ },
+ "KeywordsList": {
+ "description": "Project keywords",
+ "type": "array",
+ "items": {
+ "$ref": "#/components/schemas/Keyword"
+ },
+ "minItems": 0,
+ "example": ["project", "keywords"]
+ },
+ "Keyword": {
+ "description": "A single keyword",
+ "type": "string",
+ "minLength": 1,
+ "maxLength": 99,
+ "pattern": "^[A-Za-z0-9\\s\\-_.]*$"
+ },
+ "DataConnectorName": {
+ "description": "Renku data connector name",
+ "type": "string",
+ "minLength": 1,
+ "maxLength": 99,
+ "example": "My Remote Data :)"
+ },
+ "SourcePath": {
+ "description": "the source path to mount, usually starts with bucket/container name",
+ "type": "string",
+ "example": "bucket/my/storage/folder/"
+ },
+ "TargetPath": {
+ "description": "the target path relative to the working directory where the storage should be mounted",
+ "type": "string",
+ "example": "my/project/folder"
+ },
+ "StorageType": {
+ "description": "same as rclone prefix/ rclone config type. Ignored in requests, but returned in responses for convenience.",
+ "type": "string",
+ "readOnly": true
+ },
+ "StorageReadOnly": {
+ "description": "Whether this storage should be mounted readonly or not",
+ "type": "boolean",
+ "default": true
+ },
+ "ETag": {
+ "type": "string",
+ "description": "Entity Tag",
+ "example": "9EE498F9D565D0C41E511377425F32F3"
+ },
+ "DataConnectorsGetQuery": {
+ "description": "Query params for data connectors get request",
+ "allOf": [
+ {
+ "$ref": "#/components/schemas/PaginationRequest"
+ },
+ {
+ "properties": {
+ "namespace": {
+ "description": "A namespace, used as a filter.",
+ "type": "string",
+ "default": ""
+ }
+ }
+ }
+ ]
+ },
+ "PaginationRequest": {
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "page": {
+ "description": "Result's page number starting from 1",
+ "type": "integer",
+ "minimum": 1,
+ "default": 1
+ },
+ "per_page": {
+ "description": "The number of results per page",
+ "type": "integer",
+ "minimum": 1,
+ "maximum": 100,
+ "default": 20
+ }
+ }
+ },
+ "ErrorResponse": {
+ "type": "object",
+ "properties": {
+ "error": {
+ "type": "object",
+ "properties": {
+ "code": {
+ "type": "integer",
+ "minimum": 0,
+ "exclusiveMinimum": true,
+ "example": 1404
+ },
+ "detail": {
+ "type": "string",
+ "example": "A more detailed optional message showing what the problem was"
+ },
+ "message": {
+ "type": "string",
+ "example": "Something went wrong - please try again later"
+ }
+ },
+ "required": ["code", "message"]
+ }
+ },
+ "required": ["error"]
+ }
+ },
+ "responses": {
+ "Error": {
+ "description": "The schema for all 4xx and 5xx responses",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ }
+ },
+ "parameters": {
+ "If-Match": {
+ "in": "header",
+ "name": "If-Match",
+ "description": "If-Match header, for avoiding mid-air collisions",
+ "required": true,
+ "schema": {
+ "$ref": "#/components/schemas/ETag"
+ }
+ }
+ },
+ "securitySchemes": {
+ "oidc": {
+ "type": "openIdConnect",
+ "openIdConnectUrl": "/auth/realms/Renku/.well-known/openid-configuration"
+ }
+ }
+ },
+ "security": [
+ {
+ "oidc": ["openid"]
+ }
+ ]
+}
diff --git a/client/src/features/projectsV2/api/projectV2.enhanced-api.ts b/client/src/features/projectsV2/api/projectV2.enhanced-api.ts
index b47bdc8b3a..731211307b 100644
--- a/client/src/features/projectsV2/api/projectV2.enhanced-api.ts
+++ b/client/src/features/projectsV2/api/projectV2.enhanced-api.ts
@@ -2,6 +2,7 @@ import { AbstractKgPaginatedResponse } from "../../../utils/types/pagination.typ
import { processPaginationHeaders } from "../../../utils/helpers/kgPagination.utils";
import { projectStoragesApi as api } from "./storagesV2.api";
+
import type {
GetProjectsApiArg,
GetProjectsApiResponse as GetProjectsApiResponseOrig,
diff --git a/client/src/features/projectsV2/fields/ProjectNamespaceFormField.tsx b/client/src/features/projectsV2/fields/ProjectNamespaceFormField.tsx
index c57b47e37b..93c08fcff9 100644
--- a/client/src/features/projectsV2/fields/ProjectNamespaceFormField.tsx
+++ b/client/src/features/projectsV2/fields/ProjectNamespaceFormField.tsx
@@ -256,7 +256,7 @@ interface ProjectNamespaceControlProps {
value?: string;
}
-function ProjectNamespaceControl(props: ProjectNamespaceControlProps) {
+export function ProjectNamespaceControl(props: ProjectNamespaceControlProps) {
const { className, id, onChange, value } = props;
const dataCy = props["data-cy"];
const {
diff --git a/client/src/features/sessionsV2/DataConnectorSecretsModal.tsx b/client/src/features/sessionsV2/DataConnectorSecretsModal.tsx
index 45754a4526..a6c075957d 100644
--- a/client/src/features/sessionsV2/DataConnectorSecretsModal.tsx
+++ b/client/src/features/sessionsV2/DataConnectorSecretsModal.tsx
@@ -47,14 +47,11 @@ import { Loader } from "../../components/Loader";
import { useTestCloudStorageConnectionMutation } from "../project/components/cloudStorage/projectCloudStorage.api";
import { CLOUD_STORAGE_SAVED_SECRET_DISPLAY_VALUE } from "../project/components/cloudStorage/projectCloudStorage.constants";
-import type {
- CloudStorageDetailsOptions,
- TestCloudStorageConnectionParams,
-} from "../project/components/cloudStorage/projectCloudStorage.types";
-import { storageDefinitionFromConfig } from "../project/utils/projectCloudStorage.utils";
+import type { CloudStorageDetailsOptions } from "../project/components/cloudStorage/projectCloudStorage.types";
import type { RCloneOption } from "../projectsV2/api/storagesV2.api";
-import type { SessionStartCloudStorageConfiguration } from "./startSessionOptionsV2.types";
import { storageSecretNameToFieldName } from "../secrets/secrets.utils";
+import { DataConnectorConfiguration } from "../dataConnectorsV2/components/useDataConnectorConfiguration.hook";
+import { validationParametersFromDataConnectorConfiguration } from "../dataConnectorsV2/components/dataConnector.utils";
const CONTEXT_STRINGS = {
session: {
@@ -73,8 +70,6 @@ const CONTEXT_STRINGS = {
},
};
-export type CloudStorageConfiguration = SessionStartCloudStorageConfiguration;
-
function ClearCredentialsButton({
onSkip,
hasSavedCredentials,
@@ -98,21 +93,22 @@ function ClearCredentialsButton({
);
}
-interface CloudStorageConfigurationSecretsProps {
- cloudStorageConfig: CloudStorageConfiguration;
- context: Required;
+interface DataConnectorConfigurationSecretsProps {
+ dataConnectorConfig: DataConnectorConfiguration;
+ context: Required;
control: SensitiveFieldInputProps["control"];
}
-function CloudStorageConfigurationSecrets({
- cloudStorageConfig,
+function DataConnectorSecrets({
+ dataConnectorConfig,
context,
control,
-}: CloudStorageConfigurationSecretsProps) {
- const storage = cloudStorageConfig.cloudStorage.storage;
+}: DataConnectorConfigurationSecretsProps) {
+ const dataConnector = dataConnectorConfig.dataConnector;
+ const storage = dataConnector.storage;
const credentialFieldDict = Object.fromEntries(
- cloudStorageConfig.savedCredentialFields.map((secret) => [
+ dataConnectorConfig.savedCredentialFields.map((secret) => [
storageSecretNameToFieldName({ name: secret }),
secret,
])
@@ -122,20 +118,20 @@ function CloudStorageConfigurationSecrets({
const hasIncompleteSavedCredentials =
savedCredentialsLength > 0 &&
savedCredentialsLength !=
- cloudStorageConfig.sensitiveFieldDefinitions.length;
+ dataConnectorConfig.sensitiveFieldDefinitions.length;
return (
<>
-
{storage.name}
+
{dataConnector.name}
({storage.source_path})
- {cloudStorageConfig.sensitiveFieldDefinitions.map((field) => {
+ {dataConnectorConfig.sensitiveFieldDefinitions.map((field) => {
return (
void;
- onStart: (cloudStorageConfigs: CloudStorageConfiguration[]) => void;
- cloudStorageConfigs: CloudStorageConfiguration[] | undefined;
+ onStart: (dataConnectorConfigs: DataConnectorConfiguration[]) => void;
+ dataConnectorConfigs: DataConnectorConfiguration[] | undefined;
}
-export default function CloudStorageSecretsModal({
+export default function DataConnectorSecretsModal({
context = "session",
isOpen,
onCancel,
onStart,
- cloudStorageConfigs: initialCloudStorageConfigs,
-}: CloudStorageSecretsModalProps) {
+ dataConnectorConfigs: initialCloudStorageConfigs,
+}: DataConnectorSecretsModalProps) {
const noCredentialsConfigs = useMemo(
() =>
initialCloudStorageConfigs == null
@@ -179,7 +175,7 @@ export default function CloudStorageSecretsModal({
),
[initialCloudStorageConfigs]
);
- const [cloudStorageConfigs, setCloudStorageConfigs] = useState(
+ const [dataConnectorConfigs, setDataConnectorConfigs] = useState(
initialCloudStorageConfigs == null
? []
: initialCloudStorageConfigs.filter(
@@ -193,7 +189,7 @@ export default function CloudStorageSecretsModal({
useTestCloudStorageConnectionMutation();
const onNext = useCallback(
- (csConfigs: CloudStorageConfiguration[]) => {
+ (csConfigs: DataConnectorConfiguration[]) => {
if (index < csConfigs.length - 1) {
if (!validationResult.isUninitialized) validationResult.reset();
resetForm();
@@ -207,25 +203,26 @@ export default function CloudStorageSecretsModal({
);
const onSkip = useCallback(() => {
- if (cloudStorageConfigs.length < 1) {
+ if (dataConnectorConfigs.length < 1) {
onStart([...noCredentialsConfigs]);
return;
}
- const newCloudStorageConfigs = [...cloudStorageConfigs];
+ const newCloudStorageConfigs = [...dataConnectorConfigs];
newCloudStorageConfigs[index] = {
- ...cloudStorageConfigs[index],
+ ...dataConnectorConfigs[index],
active: false,
};
- setCloudStorageConfigs(newCloudStorageConfigs);
+ setDataConnectorConfigs(newCloudStorageConfigs);
onNext(newCloudStorageConfigs);
- }, [cloudStorageConfigs, index, noCredentialsConfigs, onNext, onStart]);
+ }, [dataConnectorConfigs, index, noCredentialsConfigs, onNext, onStart]);
const onContinue = useCallback(
(options: CloudStorageDetailsOptions) => {
- if (cloudStorageConfigs == null || cloudStorageConfigs.length < 1) return;
+ if (dataConnectorConfigs == null || dataConnectorConfigs.length < 1)
+ return;
- const config = { ...cloudStorageConfigs[index] };
+ const config = { ...dataConnectorConfigs[index] };
const sensitiveFieldValues = { ...config.sensitiveFieldValues };
const { saveCredentials } = options;
if (saveCredentials === true || saveCredentials === false) {
@@ -240,29 +237,25 @@ export default function CloudStorageSecretsModal({
});
config.sensitiveFieldValues = sensitiveFieldValues;
}
- const newStorageDetails = storageDefinitionFromConfig(config);
-
- const validateParameters: TestCloudStorageConnectionParams = {
- configuration: newStorageDetails.configuration,
- source_path: newStorageDetails.source_path,
- };
+ const validateParameters =
+ validationParametersFromDataConnectorConfiguration(config);
validateCloudStorageConnection(validateParameters);
- const newCloudStorageConfigs = [...cloudStorageConfigs];
+ const newCloudStorageConfigs = [...dataConnectorConfigs];
newCloudStorageConfigs[index] = config;
- setCloudStorageConfigs(newCloudStorageConfigs);
+ setDataConnectorConfigs(newCloudStorageConfigs);
},
- [cloudStorageConfigs, index, validateCloudStorageConnection]
+ [dataConnectorConfigs, index, validateCloudStorageConnection]
);
useEffect(() => {
- if (cloudStorageConfigs == null) return;
- if (cloudStorageConfigs[index].active && !validationResult.isSuccess)
+ if (dataConnectorConfigs == null) return;
+ if (dataConnectorConfigs[index].active && !validationResult.isSuccess)
return;
- onNext(cloudStorageConfigs);
+ onNext(dataConnectorConfigs);
}, [
- cloudStorageConfigs,
+ dataConnectorConfigs,
index,
noCredentialsConfigs,
onNext,
@@ -270,9 +263,9 @@ export default function CloudStorageSecretsModal({
validationResult,
]);
- if (cloudStorageConfigs == null) return null;
- if (cloudStorageConfigs.length < 1) return null;
- const hasSavedCredentials = cloudStorageConfigs.some(
+ if (dataConnectorConfigs == null) return null;
+ if (dataConnectorConfigs.length < 1) return null;
+ const hasSavedCredentials = dataConnectorConfigs.some(
(csc) => csc.savedCredentialFields.length > 0
);
@@ -291,8 +284,8 @@ export default function CloudStorageSecretsModal({
onSubmit={handleSubmit(onContinue)}
>
-
@@ -324,8 +317,8 @@ export default function CloudStorageSecretsModal({
}
interface CredentialsButtonsProps
- extends Pick {
- context: NonNullable;
+ extends Pick {
+ context: NonNullable;
hasSavedCredentials: boolean;
onSkip: () => void;
validationResult: ReturnType[1];
@@ -397,25 +390,25 @@ function CredentialsTestError({
}
interface ProgressBreadcrumbsProps {
- cloudStorageConfigs: CloudStorageConfiguration[];
+ dataConnectorConfigs: DataConnectorConfiguration[];
index: number;
- setCloudStorageConfigs: (configs: CloudStorageConfiguration[]) => void;
+ setDataConnectorConfigs: (configs: DataConnectorConfiguration[]) => void;
setIndex: (index: number) => void;
}
function ProgressBreadcrumbs({
- cloudStorageConfigs,
+ dataConnectorConfigs,
index,
- setCloudStorageConfigs,
+ setDataConnectorConfigs,
setIndex,
}: ProgressBreadcrumbsProps) {
- if (cloudStorageConfigs.length < 2) return null;
+ if (dataConnectorConfigs.length < 2) return null;
return (
- {cloudStorageConfigs.map((cloudStorageConfig, idx) => (
+ {dataConnectorConfigs.map((dataConnectorConfig, idx) => (
= index}
onClick={() => {
- const newCloudStorageConfigs = [...cloudStorageConfigs];
+ const newCloudStorageConfigs = [...dataConnectorConfigs];
newCloudStorageConfigs[idx] = {
- ...cloudStorageConfigs[idx],
+ ...dataConnectorConfigs[idx],
active: true,
};
- setCloudStorageConfigs(newCloudStorageConfigs);
+ setDataConnectorConfigs(newCloudStorageConfigs);
setIndex(idx);
}}
>
- {cloudStorageConfig.cloudStorage.storage.name}
+ {dataConnectorConfig.dataConnector.name}
))}
@@ -477,7 +470,7 @@ function SaveCredentialsInput({
}
interface SensitiveFieldWidgetProps
- extends CloudStorageConfigurationSecretsProps {
+ extends DataConnectorConfigurationSecretsProps {
credentialFieldDict: Record;
field: {
name: string;
@@ -487,7 +480,7 @@ interface SensitiveFieldWidgetProps
}
function SensitiveFieldWidget({
- cloudStorageConfig,
+ dataConnectorConfig,
credentialFieldDict,
context,
control,
@@ -509,7 +502,7 @@ function SensitiveFieldWidget({
}
}
const defaultValue =
- cloudStorageConfig.sensitiveFieldValues[field.name] ?? "";
+ dataConnectorConfig.sensitiveFieldValues[field.name] ?? "";
return (
) {
+ const clearButtonRef = useRef(null);
+ return (
+ <>
+
+
+ Clear
+
+
+ Forget saved credentials.
+
+
+ );
+}
+
+interface CloudStorageConfigurationSecretsProps {
+ cloudStorageConfig: CloudStorageConfiguration;
+ context: Required;
+ control: SensitiveFieldInputProps["control"];
+}
+
+function CloudStorageConfigurationSecrets({
+ cloudStorageConfig,
+ context,
+ control,
+}: CloudStorageConfigurationSecretsProps) {
+ const storage = cloudStorageConfig.cloudStorage.storage;
+
+ const credentialFieldDict = Object.fromEntries(
+ cloudStorageConfig.savedCredentialFields.map((secret) => [
+ storageSecretNameToFieldName({ name: secret }),
+ secret,
+ ])
+ );
+
+ const savedCredentialsLength = Object.keys(credentialFieldDict).length;
+ const hasIncompleteSavedCredentials =
+ savedCredentialsLength > 0 &&
+ savedCredentialsLength !=
+ cloudStorageConfig.sensitiveFieldDefinitions.length;
+
+ return (
+ <>
+
+
{storage.name}
+
({storage.source_path})
+
+ {cloudStorageConfig.sensitiveFieldDefinitions.map((field) => {
+ return (
+
+ {context === "session" &&
+ The saved credentials for this data source are incomplete so they will
+ be ignored at session launch.
+
+ )}
+
+ );
+}
+
+interface DataConnectorSecretsModalProps {
+ context?: "session" | "storage";
+ isOpen: boolean;
+ onCancel: () => void;
+ onStart: (dataConnectorConfigs: CloudStorageConfiguration[]) => void;
+ cloudStorageConfigs: CloudStorageConfiguration[] | undefined;
+}
+export default function DataConnectorSecretsModal({
+ context = "session",
+ isOpen,
+ onCancel,
+ onStart,
+ cloudStorageConfigs: initialCloudStorageConfigs,
+}: DataConnectorSecretsModalProps) {
+ const noCredentialsConfigs = useMemo(
+ () =>
+ initialCloudStorageConfigs == null
+ ? []
+ : initialCloudStorageConfigs.filter(
+ (config) => config.sensitiveFieldDefinitions.length === 0
+ ),
+ [initialCloudStorageConfigs]
+ );
+ const [dataConnectorConfigs, setDataConnectorConfigs] = useState(
+ initialCloudStorageConfigs == null
+ ? []
+ : initialCloudStorageConfigs.filter(
+ (config) => config.sensitiveFieldDefinitions.length > 0
+ )
+ );
+ const [index, setIndex] = useState(0);
+ const { control, handleSubmit, reset: resetForm } = useForm();
+
+ const [validateCloudStorageConnection, validationResult] =
+ useTestCloudStorageConnectionMutation();
+
+ const onNext = useCallback(
+ (csConfigs: CloudStorageConfiguration[]) => {
+ if (index < csConfigs.length - 1) {
+ if (!validationResult.isUninitialized) validationResult.reset();
+ resetForm();
+ setIndex((index) => index + 1);
+ } else {
+ resetForm();
+ onStart([...noCredentialsConfigs, ...csConfigs]);
+ }
+ },
+ [index, noCredentialsConfigs, onStart, resetForm, validationResult]
+ );
+
+ const onSkip = useCallback(() => {
+ if (dataConnectorConfigs.length < 1) {
+ onStart([...noCredentialsConfigs]);
+ return;
+ }
+
+ const newCloudStorageConfigs = [...dataConnectorConfigs];
+ newCloudStorageConfigs[index] = {
+ ...dataConnectorConfigs[index],
+ active: false,
+ };
+ setDataConnectorConfigs(newCloudStorageConfigs);
+ onNext(newCloudStorageConfigs);
+ }, [dataConnectorConfigs, index, noCredentialsConfigs, onNext, onStart]);
+
+ const onContinue = useCallback(
+ (options: CloudStorageDetailsOptions) => {
+ if (dataConnectorConfigs == null || dataConnectorConfigs.length < 1)
+ return;
+
+ const config = { ...dataConnectorConfigs[index] };
+ const sensitiveFieldValues = { ...config.sensitiveFieldValues };
+ const { saveCredentials } = options;
+ if (saveCredentials === true || saveCredentials === false) {
+ config.saveCredentials = saveCredentials;
+ delete options.saveCredentials;
+ }
+ if (options && Object.keys(options).length > 0) {
+ Object.entries(options).forEach(([key, value]) => {
+ if (value != undefined && value !== "") {
+ sensitiveFieldValues[key] = "" + value;
+ }
+ });
+ config.sensitiveFieldValues = sensitiveFieldValues;
+ }
+ const newStorageDetails = storageDefinitionFromConfig(config);
+
+ const validateParameters: TestCloudStorageConnectionParams = {
+ configuration: newStorageDetails.configuration,
+ source_path: newStorageDetails.source_path,
+ };
+
+ validateCloudStorageConnection(validateParameters);
+
+ const newCloudStorageConfigs = [...dataConnectorConfigs];
+ newCloudStorageConfigs[index] = config;
+ setDataConnectorConfigs(newCloudStorageConfigs);
+ },
+ [dataConnectorConfigs, index, validateCloudStorageConnection]
+ );
+
+ useEffect(() => {
+ if (dataConnectorConfigs == null) return;
+ if (dataConnectorConfigs[index].active && !validationResult.isSuccess)
+ return;
+ onNext(dataConnectorConfigs);
+ }, [
+ dataConnectorConfigs,
+ index,
+ noCredentialsConfigs,
+ onNext,
+ onStart,
+ validationResult,
+ ]);
+
+ if (dataConnectorConfigs == null) return null;
+ if (dataConnectorConfigs.length < 1) return null;
+ const hasSavedCredentials = dataConnectorConfigs.some(
+ (csc) => csc.savedCredentialFields.length > 0
+ );
+
+ return (
+
+ {CONTEXT_STRINGS[context].header}
+
+
+ );
+}
+
+interface CredentialsButtonsProps
+ extends Pick {
+ context: NonNullable;
+ hasSavedCredentials: boolean;
+ onSkip: () => void;
+ validationResult: ReturnType[1];
+}
+
+function CredentialsButtons({
+ context,
+ hasSavedCredentials,
+ onCancel,
+ onSkip,
+ validationResult,
+}: CredentialsButtonsProps) {
+ return (
+
+
+
+ {context === "session" && (
+
+ )}
+ {context === "storage" && (
+
+ )}
+
+ {validationResult.isLoading ? (
+
+ Testing
+ ) : validationResult.isError ? (
+
+
+ ) : (
+
+ {CONTEXT_STRINGS[context].continueButton}{" "}
+
+ )}
+
+
) {
+ if (!validationResult.isError) return null;
+ return (
+
+
{CONTEXT_STRINGS[context].testError}
+
+
+ {dataConnectorConfigs.map((cloudStorageConfig, idx) => (
+
+ index && "text-decoration-none"
+ )}
+ disabled={idx >= index}
+ onClick={() => {
+ const newCloudStorageConfigs = [...dataConnectorConfigs];
+ newCloudStorageConfigs[idx] = {
+ ...dataConnectorConfigs[idx],
+ active: true,
+ };
+ setDataConnectorConfigs(newCloudStorageConfigs);
+ setIndex(idx);
+ }}
+ >
+ {cloudStorageConfig.cloudStorage.storage.name}
+
+
+ ))}
+
+
+ );
+}
+
+function SaveCredentialsInput({
+ control,
+}: Pick) {
+ return (
+
+ (
+
+ Save credentials for future sessions
+
+
+ );
+}
+
+interface SensitiveFieldWidgetProps
+ extends CloudStorageConfigurationSecretsProps {
+ credentialFieldDict: Record;
+ field: {
+ name: string;
+ friendlyName: string;
+ };
+ hasIncompleteSavedCredentials: boolean;
+}
+
+function SensitiveFieldWidget({
+ cloudStorageConfig,
+ credentialFieldDict,
+ context,
+ control,
+ field,
+}: SensitiveFieldWidgetProps) {
+ const savedValue = credentialFieldDict[field.name];
+ if (context === "storage") {
+ if (savedValue != null) {
+ return (
+ ; // eslint-disable-line @typescript-eslint/no-explicit-any
+ friendlyName: string;
+ defaultValue: string | undefined;
+ option: RCloneOption;
+ showPasswordInitially?: boolean;
+}
+
+function SensitiveFieldInput({
+ control,
+ defaultValue,
+ friendlyName,
+ option,
+ showPasswordInitially = false,
+}: SensitiveFieldInputProps) {
+ const [showPassword, setShowPassword] = useState(showPasswordInitially);
+ const toggleShowPassword = useCallback(() => {
+ setShowPassword((showPassword) => !showPassword);
+ }, []);
+
+ const tooltipContainerId = `option-is-secret-${option.name}`;
+ return (
+
+
+ {friendlyName ?? option.name}
+
+
+
+
+
(
+ <>
+
+ toggleShowPassword()}
+ >
+ {showPassword ? (
+
+ Hide/show sensitive data
+
+
+
+
+ )}
+ rules={{
+ required: true,
+ validate: {
+ nonEmpty: (v) => v.trim().length > 0,
+ provided: (v) => v !== CLOUD_STORAGE_SAVED_SECRET_DISPLAY_VALUE,
+ },
+ }}
+ />
+
+ Please provide a value for {option.name}
+
+ ) {
+ const skipButtonRef = useRef(null);
+ return (
+ <>
+
+
+ Skip
+
+
+ Skip the connection test. At session launch, the storage will try to
+ mount
+ {validationResult.isError
+ ? " using the provided credentials"
+ : " without any credentials"}
+ .
+
+
+ );
+}
diff --git a/client/src/features/sessionsV2/SessionStartPage.tsx b/client/src/features/sessionsV2/SessionStartPage.tsx
index 14f2103d94..879e9349c6 100644
--- a/client/src/features/sessionsV2/SessionStartPage.tsx
+++ b/client/src/features/sessionsV2/SessionStartPage.tsx
@@ -50,8 +50,8 @@ import {
} from "../projectsV2/api/projectV2.enhanced-api";
import { storageSecretNameToFieldName } from "../secrets/secrets.utils";
import { useStartRenku2SessionMutation } from "../session/sessions.api";
-import type { CloudStorageConfiguration } from "./DataConnectorSecretsModal";
-import DataConnectorSecretsModal from "./DataConnectorSecretsModal";
+import type { CloudStorageConfiguration } from "./DataStorageSecretsModal";
+import DataStorageSecretsModal from "./DataStorageSecretsModal";
import { SelectResourceClassModal } from "./components/SessionModals/SelectResourceClass";
import { useGetProjectSessionLaunchersQuery } from "./sessionsV2.api";
import { SessionLauncher } from "./sessionsV2.types";
@@ -312,7 +312,7 @@ function StartSessionWithCloudStorageModal({
startSessionOptionsV2,
cloudStorageConfigs,
}: StartSessionWithCloudStorageModalProps) {
- const [showDataConnectorSecretsModal, setShowDataConnectorSecretsModal] =
+ const [showDataStorageSecretsModal, setShowDataConnectorSecretsModal] =
useState(false);
const dispatch = useAppDispatch();
@@ -394,8 +394,8 @@ function StartSessionWithCloudStorageModal({
title={`Starting session ${launcher.name}`}
status={steps}
/>
- (
[adminKeycloakApi.reducerPath]: adminKeycloakApi.reducer,
[adminSessionsApi.reducerPath]: adminSessionsApi.reducer,
[connectedServicesApi.reducerPath]: connectedServicesApi.reducer,
+ [dataConnectorsApi.reducerPath]: dataConnectorsApi.reducer,
[dataServicesUserApi.reducerPath]: dataServicesUserApi.reducer,
[datasetsCoreApi.reducerPath]: datasetsCoreApi.reducer,
[inactiveKgProjectsApi.reducerPath]: inactiveKgProjectsApi.reducer,
@@ -130,6 +132,7 @@ export const createStore = (
.concat(adminKeycloakApi.middleware)
.concat(adminSessionsApi.middleware)
.concat(connectedServicesApi.middleware)
+ .concat(dataConnectorsApi.middleware)
// this is causing some problems, and I do not know why
.concat(dataServicesUserApi.middleware)
.concat(datasetsCoreApi.middleware)
diff --git a/tests/cypress/e2e/groupV2.spec.ts b/tests/cypress/e2e/groupV2.spec.ts
index 3dfa0e661b..e6ee1b0e9d 100644
--- a/tests/cypress/e2e/groupV2.spec.ts
+++ b/tests/cypress/e2e/groupV2.spec.ts
@@ -93,6 +93,20 @@ describe("Edit v2 group", () => {
cy.visit("/v2/groups");
});
+ it("shows a group", () => {
+ fixtures
+ .readGroupV2()
+ .readGroupV2Namespace()
+ .listGroupV2Members()
+ .listProjectV2ByNamespace()
+ .listDataConnectors({ namespace: "test-2-group-v2" });
+ cy.contains("List Groups").should("be.visible");
+ cy.contains("test 2 group-v2").should("be.visible").click();
+ cy.wait("@readGroupV2");
+ cy.contains("test 2 group-v2").should("be.visible");
+ cy.contains("public-storage").should("be.visible");
+ });
+
it("allows editing group metadata", () => {
fixtures
.readGroupV2()
@@ -199,3 +213,126 @@ describe("Edit v2 group", () => {
cy.contains("Return to the groups list").click();
});
});
+
+describe("Work with group data connectors", () => {
+ beforeEach(() => {
+ fixtures
+ .config()
+ .versions()
+ .userTest()
+ .dataServicesUser({
+ response: { id: "0945f006-e117-49b7-8966-4c0842146313" },
+ })
+ .namespaces()
+ .listNamespaceV2();
+ fixtures.projects().landingUserProjects().listGroupV2();
+ cy.visit("/v2/groups");
+ });
+
+ it("shows group data connectors", () => {
+ fixtures
+ .readGroupV2()
+ .readGroupV2Namespace()
+ .listGroupV2Members()
+ .listProjectV2ByNamespace()
+ .listDataConnectors({ namespace: "test-2-group-v2" });
+ cy.contains("List Groups").should("be.visible");
+ cy.contains("test 2 group-v2").should("be.visible").click();
+ cy.wait("@readGroupV2");
+ cy.contains("test 2 group-v2").should("be.visible");
+ cy.contains("public-storage").should("be.visible");
+ });
+
+ it("add a group data connector", () => {
+ fixtures
+ .getStorageSchema({ fixture: "cloudStorage/storage-schema-s3.json" })
+ .readGroupV2()
+ .readGroupV2Namespace()
+ .listGroupV2Members()
+ .listProjectV2ByNamespace()
+ .listDataConnectors({ namespace: "test-2-group-v2" })
+ .testCloudStorage({ success: false })
+ .postDataConnector({ namespace: "test-2-group-v2" });
+ cy.contains("test 2 group-v2").should("be.visible").click();
+ cy.wait("@readGroupV2");
+ cy.contains("public-storage").should("be.visible");
+ cy.getDataCy("add-data-connector").should("be.visible").click();
+ // Pick a provider
+ cy.getDataCy("data-storage-s3").click();
+ cy.getDataCy("data-provider-AWS").click();
+ cy.getDataCy("data-connector-edit-next-button").click();
+
+ // Fill out the details
+ cy.get("#sourcePath").type("bucket/my-source");
+ cy.get("#access_key_id").type("access key");
+ cy.get("#secret_access_key").type("secret key");
+ cy.getDataCy("test-data-connector-button").click();
+ cy.getDataCy("add-data-connector-continue-button").contains("Skip").click();
+ cy.getDataCy("cloud-storage-edit-mount").within(() => {
+ cy.get("#name").type("example storage without credentials");
+ });
+ cy.getDataCy("data-connector-edit-update-button").click();
+ cy.wait("@postDataConnector");
+ cy.getDataCy("cloud-storage-edit-body").should(
+ "contain.text",
+ "The data connector test-2-group-v2/example-storage-without-credentials has been successfully added."
+ );
+ cy.getDataCy("data-connector-edit-close-button").click();
+ cy.wait("@getDataConnectors");
+ });
+
+ it("edit a group data connector", () => {
+ fixtures
+ .getStorageSchema({ fixture: "cloudStorage/storage-schema-s3.json" })
+ .readGroupV2()
+ .readGroupV2Namespace()
+ .listGroupV2Members()
+ .listProjectV2ByNamespace()
+ .listDataConnectors({ namespace: "test-2-group-v2" })
+ .testCloudStorage({ success: true })
+ .patchDataConnector({ namespace: "test-2-group-v2" });
+ cy.contains("test 2 group-v2").should("be.visible").click();
+ cy.wait("@readGroupV2");
+ cy.contains("public-storage").should("be.visible").click();
+ cy.getDataCy("data-connector-edit").should("be.visible").click();
+ // Fill out the details
+ cy.getDataCy("test-data-connector-button").click();
+ cy.getDataCy("add-data-connector-continue-button")
+ .contains("Continue")
+ .click();
+ cy.getDataCy("data-connector-edit-update-button").click();
+ cy.wait("@patchDataConnector");
+ cy.getDataCy("cloud-storage-edit-body").should(
+ "contain.text",
+ "The data connector test-2-group-v2/public-storage has been successfully updated."
+ );
+ cy.getDataCy("data-connector-edit-close-button").click();
+ cy.wait("@getDataConnectors");
+ });
+
+ it("delete a group data connector", () => {
+ fixtures
+ .getStorageSchema({ fixture: "cloudStorage/storage-schema-s3.json" })
+ .readGroupV2()
+ .readGroupV2Namespace()
+ .listGroupV2Members()
+ .listProjectV2ByNamespace()
+ .listDataConnectors({ namespace: "test-2-group-v2" })
+ .testCloudStorage({ success: true })
+ .deleteDataConnector();
+ cy.contains("test 2 group-v2").should("be.visible").click();
+ cy.wait("@readGroupV2");
+ cy.contains("public-storage").should("be.visible").click();
+ cy.getDataCy("button-with-menu-dropdown").should("be.visible").click();
+ cy.getDataCy("data-connector-delete").should("be.visible").click();
+ cy.contains("Are you sure you want to delete this data connector").should(
+ "be.visible"
+ );
+ cy.getDataCy("delete-confirmation-input").clear().type("public-storage");
+ cy.getDataCy("delete-data-connector-modal-button")
+ .should("be.visible")
+ .click();
+ cy.wait("@deleteDataConnector");
+ cy.wait("@getDataConnectors");
+ });
+});
diff --git a/tests/cypress/fixtures/dataConnector/data-connector-multiple.json b/tests/cypress/fixtures/dataConnector/data-connector-multiple.json
new file mode 100644
index 0000000000..54a30b8f15
--- /dev/null
+++ b/tests/cypress/fixtures/dataConnector/data-connector-multiple.json
@@ -0,0 +1,105 @@
+[
+ {
+ "id": "ULID-2",
+ "name": "public-storage",
+ "namespace": "user1-uuid",
+ "slug": "public-storage",
+ "storage": {
+ "storage_type": "s3",
+ "configuration": {
+ "type": "s3",
+ "provider": "Other",
+ "endpoint": "https://s3.example.com"
+ },
+ "source_path": "bucket/source",
+ "target_path": "external_storage/public",
+ "readonly": true,
+ "sensitive_fields": []
+ },
+ "creation_date": "2023-11-15T09:55:59Z",
+ "created_by": { "id": "user1-uuid" },
+ "visibility": "public",
+ "description": "Data connector 2 description"
+ },
+ {
+ "id": "ULID-3",
+ "name": "private-storage-1",
+ "namespace": "user1-uuid",
+ "slug": "private-storage-1",
+ "storage": {
+ "storage_type": "s3",
+ "configuration": {
+ "type": "s3",
+ "provider": "AWS",
+ "access_key_id": "",
+ "secret_access_key": ""
+ },
+ "source_path": "bucket/my-source-1",
+ "target_path": "external_storage/private-1",
+ "readonly": true,
+ "sensitive_fields": [
+ {
+ "name": "access_key_id",
+ "help": "AWS Access Key ID.\n\nLeave blank for anonymous access or runtime credentials.",
+ "provider": "",
+ "default": "",
+ "default_str": "",
+ "required": false,
+ "sensitive": true,
+ "advanced": false,
+ "exclusive": false
+ },
+ {
+ "name": "secret_access_key",
+ "help": "AWS Secret Access Key (password).\n\nLeave blank for anonymous access or runtime credentials.",
+ "provider": "",
+ "default": "",
+ "default_str": "",
+ "required": false,
+ "sensitive": true,
+ "advanced": false,
+ "exclusive": false
+ }
+ ]
+ },
+ "creation_date": "2023-11-15T09:55:59Z",
+ "created_by": { "id": "user1-uuid" },
+ "visibility": "private",
+ "description": "Data connector 3 description"
+ },
+ {
+ "id": "ULID-4",
+ "name": "webdav",
+ "namespace": "user1-uuid",
+ "slug": "webdav",
+ "storage": {
+ "storage_type": "webdav",
+ "configuration": {
+ "url": "https://s3-thing.com/",
+ "pass": "",
+ "type": "webdav",
+ "user": "cramakri"
+ },
+ "source_path": "/",
+ "target_path": "external_storage/webdav",
+ "readonly": true,
+ "sensitive_fields": [
+ {
+ "name": "pass",
+ "help": "Password.",
+ "provider": "",
+ "default": "",
+ "default_str": "",
+ "required": false,
+ "sensitive": true,
+ "advanced": false,
+ "exclusive": false
+ }
+ ]
+ },
+ "creation_date": "2023-11-15T09:55:59Z",
+ "created_by": { "id": "user1-uuid" },
+ "visibility": "private",
+ "description": "Data connector 3 description"
+ }
+]
diff --git a/tests/cypress/fixtures/dataConnector/data-connector-secrets-empty.json b/tests/cypress/fixtures/dataConnector/data-connector-secrets-empty.json
new file mode 100644
index 0000000000..fe51488c70
--- /dev/null
+++ b/tests/cypress/fixtures/dataConnector/data-connector-secrets-empty.json
@@ -0,0 +1 @@
+[]
diff --git a/tests/cypress/fixtures/dataConnector/data-connector-secrets-partial.json b/tests/cypress/fixtures/dataConnector/data-connector-secrets-partial.json
new file mode 100644
index 0000000000..b74beb578b
--- /dev/null
+++ b/tests/cypress/fixtures/dataConnector/data-connector-secrets-partial.json
@@ -0,0 +1,6 @@
+[
+ {
+ "name": "secret_access_key",
+ "secret_id": "ULID1"
+ }
+]
diff --git a/tests/cypress/fixtures/dataConnector/data-connector-secrets.json b/tests/cypress/fixtures/dataConnector/data-connector-secrets.json
new file mode 100644
index 0000000000..576c8f2acd
--- /dev/null
+++ b/tests/cypress/fixtures/dataConnector/data-connector-secrets.json
@@ -0,0 +1,10 @@
+[
+ {
+ "name": "access_key_id",
+ "secret_id": "ULID2"
+ },
+ {
+ "name": "secret_access_key",
+ "secret_id": "ULID1"
+ }
+]
diff --git a/tests/cypress/fixtures/dataConnector/data-connector-with-secrets-values-empty.json b/tests/cypress/fixtures/dataConnector/data-connector-with-secrets-values-empty.json
new file mode 100644
index 0000000000..387f54769f
--- /dev/null
+++ b/tests/cypress/fixtures/dataConnector/data-connector-with-secrets-values-empty.json
@@ -0,0 +1,44 @@
+[
+ {
+ "storage": {
+ "configuration": {
+ "type": "s3",
+ "provider": "AWS",
+ "access_key_id": "",
+ "secret_access_key": ""
+ },
+ "name": "example-storage",
+ "project_id": 1,
+ "readonly": true,
+ "source_path": "bucket/my-source",
+ "storage_id": "2",
+ "storage_type": "s3",
+ "target_path": "external_storage/aws"
+ },
+ "sensitive_fields": [
+ {
+ "name": "access_key_id",
+ "help": "AWS Access Key ID.\n\nLeave blank for anonymous access or runtime credentials.",
+ "provider": "",
+ "default": "",
+ "default_str": "",
+ "required": false,
+ "sensitive": true,
+ "advanced": false,
+ "exclusive": false
+ },
+ {
+ "name": "secret_access_key",
+ "help": "AWS Secret Access Key (password).\n\nLeave blank for anonymous access or runtime credentials.",
+ "provider": "",
+ "default": "",
+ "default_str": "",
+ "required": false,
+ "sensitive": true,
+ "advanced": false,
+ "exclusive": false
+ }
+ ],
+ "secrets": []
+ }
+]
diff --git a/tests/cypress/fixtures/dataConnector/data-connector-with-secrets-values-full.json b/tests/cypress/fixtures/dataConnector/data-connector-with-secrets-values-full.json
new file mode 100644
index 0000000000..b2b4e0561c
--- /dev/null
+++ b/tests/cypress/fixtures/dataConnector/data-connector-with-secrets-values-full.json
@@ -0,0 +1,53 @@
+[
+ {
+ "storage": {
+ "configuration": {
+ "type": "s3",
+ "provider": "AWS",
+ "access_key_id": "",
+ "secret_access_key": ""
+ },
+ "name": "example-storage",
+ "project_id": 1,
+ "readonly": true,
+ "source_path": "bucket/my-source",
+ "storage_id": "2",
+ "storage_type": "s3",
+ "target_path": "external_storage/aws"
+ },
+ "sensitive_fields": [
+ {
+ "name": "access_key_id",
+ "help": "AWS Access Key ID.\n\nLeave blank for anonymous access or runtime credentials.",
+ "provider": "",
+ "default": "",
+ "default_str": "",
+ "required": false,
+ "sensitive": true,
+ "advanced": false,
+ "exclusive": false
+ },
+ {
+ "name": "secret_access_key",
+ "help": "AWS Secret Access Key (password).\n\nLeave blank for anonymous access or runtime credentials.",
+ "provider": "",
+ "default": "",
+ "default_str": "",
+ "required": false,
+ "sensitive": true,
+ "advanced": false,
+ "exclusive": false
+ }
+ ],
+ "secrets": [
+ {
+ "name": "access_key_id",
+ "secret_id": "ULID2"
+ },
+ {
+ "name": "secret_access_key",
+ "secret_id": "ULID1"
+ }
+ ]
+ }
+]
diff --git a/tests/cypress/fixtures/dataConnector/data-connector-with-secrets-values-partial.json b/tests/cypress/fixtures/dataConnector/data-connector-with-secrets-values-partial.json
new file mode 100644
index 0000000000..be3c74eef1
--- /dev/null
+++ b/tests/cypress/fixtures/dataConnector/data-connector-with-secrets-values-partial.json
@@ -0,0 +1,49 @@
+[
+ {
+ "storage": {
+ "configuration": {
+ "type": "s3",
+ "provider": "AWS",
+ "access_key_id": "",
+ "secret_access_key": ""
+ },
+ "name": "example-storage",
+ "project_id": 1,
+ "readonly": true,
+ "source_path": "bucket/my-source",
+ "storage_id": "2",
+ "storage_type": "s3",
+ "target_path": "external_storage/aws"
+ },
+ "sensitive_fields": [
+ {
+ "name": "access_key_id",
+ "help": "AWS Access Key ID.\n\nLeave blank for anonymous access or runtime credentials.",
+ "provider": "",
+ "default": "",
+ "default_str": "",
+ "required": false,
+ "sensitive": true,
+ "advanced": false,
+ "exclusive": false
+ },
+ {
+ "name": "secret_access_key",
+ "help": "AWS Secret Access Key (password).\n\nLeave blank for anonymous access or runtime credentials.",
+ "provider": "",
+ "default": "",
+ "default_str": "",
+ "required": false,
+ "sensitive": true,
+ "advanced": false,
+ "exclusive": false
+ }
+ ],
+ "secrets": [
+ {
+ "name": "secret_access_key",
+ "secret_id": "ULID1"
+ }
+ ]
+ }
+]
diff --git a/tests/cypress/fixtures/dataConnector/data-connector-with-secrets.json b/tests/cypress/fixtures/dataConnector/data-connector-with-secrets.json
new file mode 100644
index 0000000000..8c16dfcd73
--- /dev/null
+++ b/tests/cypress/fixtures/dataConnector/data-connector-with-secrets.json
@@ -0,0 +1,43 @@
+[
+ {
+ "storage": {
+ "configuration": {
+ "type": "s3",
+ "provider": "AWS",
+ "access_key_id": "",
+ "secret_access_key": ""
+ },
+ "name": "example-storage",
+ "project_id": 1,
+ "readonly": true,
+ "source_path": "bucket/my-source",
+ "storage_id": "2",
+ "storage_type": "s3",
+ "target_path": "external_storage/aws"
+ },
+ "sensitive_fields": [
+ {
+ "name": "access_key_id",
+ "help": "AWS Access Key ID.\n\nLeave blank for anonymous access or runtime credentials.",
+ "provider": "",
+ "default": "",
+ "default_str": "",
+ "required": false,
+ "sensitive": true,
+ "advanced": false,
+ "exclusive": false
+ },
+ {
+ "name": "secret_access_key",
+ "help": "AWS Secret Access Key (password).\n\nLeave blank for anonymous access or runtime credentials.",
+ "provider": "",
+ "default": "",
+ "default_str": "",
+ "required": false,
+ "sensitive": true,
+ "advanced": false,
+ "exclusive": false
+ }
+ ]
+ }
+]
diff --git a/tests/cypress/fixtures/dataConnector/data-connector.json b/tests/cypress/fixtures/dataConnector/data-connector.json
new file mode 100644
index 0000000000..0e2548b724
--- /dev/null
+++ b/tests/cypress/fixtures/dataConnector/data-connector.json
@@ -0,0 +1,22 @@
+{
+ "id": "ULID-1",
+ "name": "example-storage",
+ "namespace": "user1-uuid",
+ "slug": "example-storage",
+ "storage": {
+ "storage_type": "s3",
+ "configuration": {
+ "type": "s3",
+ "provider": "Other",
+ "endpoint": "https://s3.example.com"
+ },
+ "source_path": "bucket/source",
+ "target_path": "mount/path",
+ "readonly": true,
+ "sensitive_fields": []
+ },
+ "creation_date": "2023-11-15T09:55:59Z",
+ "created_by": { "id": "user1-uuid" },
+ "visibility": "public",
+ "description": "Data connector 1 description"
+}
diff --git a/tests/cypress/fixtures/dataConnector/new-data-connector.json b/tests/cypress/fixtures/dataConnector/new-data-connector.json
new file mode 100644
index 0000000000..49ca061ff4
--- /dev/null
+++ b/tests/cypress/fixtures/dataConnector/new-data-connector.json
@@ -0,0 +1,25 @@
+{
+ "id": "ULID-5",
+ "name": "example-storage",
+ "namespace": "user1-uuid",
+ "slug": "example-storage",
+ "storage": {
+ "storage_type": "s3",
+ "configuration": {
+ "provider": "AWS",
+ "region": "eu-central-2"
+ },
+
+ "source_path": "bucket/my-source",
+ "target_path": "external_storage/aws",
+ "readonly": false,
+ "sensitive_fields": [
+ { "help": "First credential", "name": "first" },
+ { "help": "Second credential", "name": "second" }
+ ]
+ },
+ "creation_date": "2023-11-15T09:55:59Z",
+ "created_by": { "id": "user1-uuid" },
+ "visibility": "private",
+ "description": "Data connector 5 description"
+}
diff --git a/tests/cypress/support/renkulab-fixtures/dataConnectors.ts b/tests/cypress/support/renkulab-fixtures/dataConnectors.ts
new file mode 100644
index 0000000000..65927428e0
--- /dev/null
+++ b/tests/cypress/support/renkulab-fixtures/dataConnectors.ts
@@ -0,0 +1,125 @@
+/*!
+ * Copyright 2023 - Swiss Data Science Center (SDSC)
+ * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+ * Eidgenössische Technische Hochschule Zürich (ETHZ).
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { FixturesConstructor } from "./fixtures";
+import { SimpleFixture } from "./fixtures.types";
+
+/**
+ * Fixtures for Cloud Storage
+ */
+
+interface DataConnectorArgs extends SimpleFixture {
+ namespace?: string;
+ visibility?: string;
+}
+
+export function DataConnector(Parent: T) {
+ return class DataConnectorFixtures extends Parent {
+ listDataConnectors(args?: DataConnectorArgs) {
+ const {
+ fixture = "dataConnector/data-connector-multiple.json",
+ name = "getDataConnectors",
+ namespace,
+ } = args ?? {};
+ cy.fixture(fixture).then((dcs) => {
+ // eslint-disable-next-line max-nested-callbacks
+ cy.intercept(
+ "GET",
+ `/ui-server/api/data/data_connectors?namespace=${namespace}*`,
+ (req) => {
+ const response = dcs.map((dc) => {
+ return {
+ ...dc,
+ namespace,
+ };
+ });
+ req.reply({ body: response });
+ }
+ ).as(name);
+ });
+ return this;
+ }
+
+ postDataConnector(args?: DataConnectorArgs) {
+ const {
+ fixture = "dataConnector/new-data-connector.json",
+ name = "postDataConnector",
+ namespace,
+ visibility = "private",
+ } = args ?? {};
+ cy.fixture(fixture).then((dataConnector) => {
+ // eslint-disable-next-line max-nested-callbacks
+ cy.intercept("POST", "/ui-server/api/data/data_connectors", (req) => {
+ const newDataConnector = req.body;
+ expect(newDataConnector.namespace).to.not.be.undefined;
+ expect(newDataConnector.slug).to.not.be.undefined;
+ expect(newDataConnector.visibility).to.not.be.undefined;
+ expect(newDataConnector.visibility).equal(visibility);
+ if (namespace) {
+ expect(newDataConnector.namespace).equal(namespace);
+ }
+ dataConnector.namespace = newDataConnector.namespace;
+ dataConnector.slug = newDataConnector.slug;
+ dataConnector.visibility = newDataConnector.visibility;
+ req.reply({ body: dataConnector, statusCode: 201, delay: 1000 });
+ }).as(name);
+ });
+ return this;
+ }
+
+ patchDataConnector(args?: DataConnectorArgs) {
+ const {
+ fixture = "dataConnector/new-data-connector.json",
+ name = "patchDataConnector",
+ namespace,
+ } = args ?? {};
+ cy.fixture(fixture).then((dataConnector) => {
+ // eslint-disable-next-line max-nested-callbacks
+ cy.intercept(
+ "PATCH",
+ "/ui-server/api/data/data_connectors/*",
+ (req) => {
+ const newDataConnector = req.body;
+ expect(newDataConnector.namespace).to.not.be.undefined;
+ expect(newDataConnector.slug).to.not.be.undefined;
+ expect(newDataConnector.visibility).to.not.be.undefined;
+ if (namespace) {
+ expect(newDataConnector.namespace).equal(namespace);
+ }
+ dataConnector.namespace = newDataConnector.namespace;
+ dataConnector.slug = newDataConnector.slug;
+ dataConnector.visibility = newDataConnector.visibility;
+ req.reply({ body: dataConnector, statusCode: 201, delay: 1000 });
+ }
+ ).as(name);
+ });
+ return this;
+ }
+
+ deleteDataConnector(args?: DataConnectorArgs) {
+ const { name = "deleteDataConnector" } = args ?? {};
+ const response = { statusCode: 204 };
+ cy.intercept(
+ "DELETE",
+ "/ui-server/api/data/data_connectors/*",
+ response
+ ).as(name);
+ return this;
+ }
+ };
+}
diff --git a/tests/cypress/support/renkulab-fixtures/index.ts b/tests/cypress/support/renkulab-fixtures/index.ts
index b1000cabff..abeedf8d16 100644
--- a/tests/cypress/support/renkulab-fixtures/index.ts
+++ b/tests/cypress/support/renkulab-fixtures/index.ts
@@ -22,6 +22,7 @@
import { Admin } from "./admin";
import { CloudStorage } from "./cloudStorage";
import { Dashboard } from "./dashboard";
+import { DataConnector } from "./dataConnectors";
import { DataServices } from "./dataServices";
import { Datasets } from "./datasets";
import BaseFixtures from "./fixtures";
@@ -45,17 +46,19 @@ const V1Fixtures = NewProject(
Dashboard(
Sessions(
Admin(
- DataServices(
- CloudStorage(
- Datasets(
- Projects(
- ProjectV2(
- SearchV2(
- Secrets(
- Terms(
- User(
- UserPreferences(
- Workflows(KgSearch(Global(BaseFixtures)))
+ DataConnector(
+ DataServices(
+ CloudStorage(
+ Datasets(
+ Projects(
+ ProjectV2(
+ SearchV2(
+ Secrets(
+ Terms(
+ User(
+ UserPreferences(
+ Workflows(KgSearch(Global(BaseFixtures)))
+ )
)
)
)
diff --git a/tests/cypress/support/renkulab-fixtures/projectV2.ts b/tests/cypress/support/renkulab-fixtures/projectV2.ts
index cfaa4048f4..6086ef5061 100644
--- a/tests/cypress/support/renkulab-fixtures/projectV2.ts
+++ b/tests/cypress/support/renkulab-fixtures/projectV2.ts
@@ -153,6 +153,27 @@ export function ProjectV2(Parent: T) {
return this;
}
+ listProjectV2ByNamespace(args?: Omit) {
+ const {
+ fixture = "projectV2/list-projectV2.json",
+ name = "listProjectV2ByNamespace",
+ namespace = "test-2-group-v2",
+ } = args ?? {};
+ cy.fixture(fixture).then((content) => {
+ const result = content.map((project) => {
+ project.namespace = namespace;
+ return project;
+ });
+ const response = { body: result };
+ cy.intercept(
+ "GET",
+ `/ui-server/api/data/projects?namespace=${namespace}*`,
+ response
+ ).as(name);
+ });
+ return this;
+ }
+
listProjectV2Members(args?: ListProjectV2MembersFixture) {
const {
fixture = "projectV2/list-projectV2-members.json",