Kubeflow Installation

Table of Contents

1. Prerequisites
2. Setup Database and Tables for Kubeflow
3. Install Kubeflow
4. Create Kubeflow User

---

Detailed Steps

For detailed explanations of {variables}, refer to the Terminology page

1. Prerequisites

Ensure the setup for the installation environment is complete. (Refer to 1. Setup Deploy Environment)

---

2. Setup Database and Tables for Kubeflow

• Create the kubeflow-table.sql file.

  [Expand kubeflow-table.sql]

  cat <<EOT > kubeflow-table.sql
  CREATE DATABASE IF NOT EXISTS ai_conductor;
  CREATE DATABASE IF NOT EXISTS cachedb;
  CREATE DATABASE IF NOT EXISTS edge_conductor;
  CREATE DATABASE IF NOT EXISTS kubeflow;
  CREATE DATABASE IF NOT EXISTS metadb;
  CREATE DATABASE IF NOT EXISTS mlpipeline;
  
  create table if not exists metadb.Artifact
  (
      id                           int auto_increment
          primary key,
      type_id                      int              not null,
      uri                          text             null,
      state                        int              null,
      name                         varchar(255)     null,
      create_time_since_epoch      bigint default 0 not null,
      last_update_time_since_epoch bigint default 0 not null,
      external_id                  varchar(255)     null,
      constraint UniqueArtifactExternalId
          unique (external_id),
      constraint UniqueArtifactTypeName
          unique (type_id, name)
  );
  
  create index idx_artifact_create_time_since_epoch
      on metadb.Artifact (create_time_since_epoch);
  
  create index idx_artifact_external_id
      on metadb.Artifact (external_id);
  
  create index idx_artifact_last_update_time_since_epoch
      on metadb.Artifact (last_update_time_since_epoch);
  
  create index idx_artifact_uri
      on metadb.Artifact (uri(255));
  
  create table if not exists metadb.ArtifactProperty
  (
      artifact_id        int          not null,
      name               varchar(255) not null,
      is_custom_property tinyint(1)   not null,
      int_value          int          null,
      double_value       double       null,
      string_value       mediumtext   null,
      byte_value         mediumblob   null,
      proto_value        mediumblob   null,
      bool_value         tinyint(1)   null,
      primary key (artifact_id, name, is_custom_property)
  );
  
  create index idx_artifact_property_double
      on metadb.ArtifactProperty (name, is_custom_property, double_value);
  
  create index idx_artifact_property_int
      on metadb.ArtifactProperty (name, is_custom_property, int_value);
  
  create index idx_artifact_property_string
      on metadb.ArtifactProperty (name, is_custom_property, string_value(255));
  
  create table if not exists metadb.Association
  (
      id           int auto_increment
          primary key,
      context_id   int not null,
      execution_id int not null,
      constraint context_id
          unique (context_id, execution_id)
  );
  
  create table if not exists metadb.Attribution
  (
      id          int auto_increment
          primary key,
      context_id  int not null,
      artifact_id int not null,
      constraint context_id
          unique (context_id, artifact_id)
  );
  
  create table if not exists metadb.Context
  (
      id                           int auto_increment
          primary key,
      type_id                      int              not null,
      name                         varchar(255)     not null,
      create_time_since_epoch      bigint default 0 not null,
      last_update_time_since_epoch bigint default 0 not null,
      external_id                  varchar(255)     null,
      constraint UniqueContextExternalId
          unique (external_id),
      constraint type_id
          unique (type_id, name)
  );
  
  create index idx_context_create_time_since_epoch
      on metadb.Context (create_time_since_epoch);
  
  create index idx_context_external_id
      on metadb.Context (external_id);
  
  create index idx_context_last_update_time_since_epoch
      on metadb.Context (last_update_time_since_epoch);
  
  create table if not exists metadb.ContextProperty
  (
      context_id         int          not null,
      name               varchar(255) not null,
      is_custom_property tinyint(1)   not null,
      int_value          int          null,
      double_value       double       null,
      string_value       mediumtext   null,
      byte_value         mediumblob   null,
      proto_value        mediumblob   null,
      bool_value         tinyint(1)   null,
      primary key (context_id, name, is_custom_property)
  );
  
  create index idx_context_property_double
      on metadb.ContextProperty (name, is_custom_property, double_value);
  
  create index idx_context_property_int
      on metadb.ContextProperty (name, is_custom_property, int_value);
  
  create index idx_context_property_string
      on metadb.ContextProperty (name, is_custom_property, string_value(255));
  
  create table if not exists metadb.Event
  (
      id                       int auto_increment
          primary key,
      artifact_id              int    not null,
      execution_id             int    not null,
      type                     int    not null,
      milliseconds_since_epoch bigint null,
      constraint UniqueEvent
          unique (artifact_id, execution_id, type)
  );
  
  create index idx_event_execution_id
      on metadb.Event (execution_id);
  
  create table if not exists metadb.EventPath
  (
      event_id      int        not null,
      is_index_step tinyint(1) not null,
      step_index    int        null,
      step_key      text       null
  );
  
  create index idx_eventpath_event_id
      on metadb.EventPath (event_id);
  
  create table if not exists metadb.Execution
  (
      id                           int auto_increment
          primary key,
      type_id                      int              not null,
      last_known_state             int              null,
      name                         varchar(255)     null,
      create_time_since_epoch      bigint default 0 not null,
      last_update_time_since_epoch bigint default 0 not null,
      external_id                  varchar(255)     null,
      constraint UniqueExecutionExternalId
          unique (external_id),
      constraint UniqueExecutionTypeName
          unique (type_id, name)
  );
  
  create index idx_execution_create_time_since_epoch
      on metadb.Execution (create_time_since_epoch);
  
  create index idx_execution_external_id
      on metadb.Execution (external_id);
  
  create index idx_execution_last_update_time_since_epoch
      on metadb.Execution (last_update_time_since_epoch);
  
  create table if not exists metadb.ExecutionProperty
  (
      execution_id       int          not null,
      name               varchar(255) not null,
      is_custom_property tinyint(1)   not null,
      int_value          int          null,
      double_value       double       null,
      string_value       mediumtext   null,
      byte_value         mediumblob   null,
      proto_value        mediumblob   null,
      bool_value         tinyint(1)   null,
      primary key (execution_id, name, is_custom_property)
  );
  
  create index idx_execution_property_double
      on metadb.ExecutionProperty (name, is_custom_property, double_value);
  
  create index idx_execution_property_int
      on metadb.ExecutionProperty (name, is_custom_property, int_value);
  
  create index idx_execution_property_string
      on metadb.ExecutionProperty (name, is_custom_property, string_value(255));
  
  create table if not exists metadb.MLMDEnv
  (
      schema_version int not null
          primary key
  );
  
  insert into metadb.MLMDEnv (schema_version) value(10);
  
  create table if not exists metadb.ParentContext
  (
      context_id        int not null,
      parent_context_id int not null,
      primary key (context_id, parent_context_id)
  );
  
  create index idx_parentcontext_parent_context_id
      on metadb.ParentContext (parent_context_id);
  
  create table if not exists metadb.ParentType
  (
      type_id        int not null,
      parent_type_id int not null,
      primary key (type_id, parent_type_id)
  );
  
  create table if not exists metadb.Type
  (
      id          int auto_increment
          primary key,
      name        varchar(255) not null,
      version     varchar(255) null,
      type_kind   tinyint(1)   not null,
      description text         null,
      input_type  text         null,
      output_type text         null,
      external_id varchar(255) null,
      constraint UniqueTypeExternalId
          unique (external_id)
  );
  
  create index idx_type_external_id
      on metadb.Type (external_id);
  
  create index idx_type_name
      on metadb.Type (name);
  
  create table if not exists metadb.TypeProperty
  (
      type_id   int          not null,
      name      varchar(255) not null,
      data_type int          null,
  
  
      primary key (type_id, name)
  );
  
  create table if not exists mlpipeline.db_statuses
  (
      HaveSamplesLoaded tinyint(1) not null
          primary key
  );
  
  create table if not exists mlpipeline.default_experiments
  (
      DefaultExperimentId varchar(255) not null
          primary key
  );
  
  create table if not exists cachedb.execution_caches
  (
      ID                bigint auto_increment
          primary key,
      ExecutionCacheKey varchar(255) not null,
      ExecutionTemplate longtext     not null,
      ExecutionOutput   longtext     null,
      MaxCacheStaleness bigint       not null,
      StartedAtInSec    bigint       not null,
      EndedAtInSec      bigint       not null
  );
  
  create index idx_cache_key
      on cachedb.execution_caches (ExecutionCacheKey);
  
  create table if not exists mlpipeline.experiments
  (
      UUID           varchar(255) not null
          primary key,
      Name           varchar(255) not null,
      Description    varchar(255) not null,
      CreatedAtInSec bigint       not null,
      Namespace      varchar(255) not null,
      StorageState   varchar(255) not null,
      constraint idx_name_namespace
          unique (Name, Namespace)
  );
  
  create table if not exists mlpipeline.jobs
  (
      UUID                           varchar(255) not null
          primary key,
      DisplayName                    varchar(255) not null,
      Name                           varchar(255) not null,
      Namespace                      varchar(255) not null,
      ServiceAccount                 varchar(255) not null,
      Description                    varchar(255) not null,
      MaxConcurrency                 bigint       not null,
      NoCatchup                      tinyint(1)   not null,
      CreatedAtInSec                 bigint       not null,
      UpdatedAtInSec                 bigint       not null,
      Enabled                        tinyint(1)   not null,
      CronScheduleStartTimeInSec     bigint       null,
      CronScheduleEndTimeInSec       bigint       null,
      Schedule                       varchar(255) null,
      PeriodicScheduleStartTimeInSec bigint       null,
      PeriodicScheduleEndTimeInSec   bigint       null,
      IntervalSecond                 bigint       null,
      PipelineId                     varchar(255) not null,
      PipelineName                   varchar(255) not null,
      PipelineSpecManifest           longtext     null,
      WorkflowSpecManifest           longtext     not null,
      Parameters                     longtext     null,
      RuntimeParameters              longtext     null,
      PipelineRoot                   longtext     null,
      Conditions                     varchar(255) not null,
      ExperimentUUID                 varchar(255) not null,
      PipelineVersionId              varchar(255) null
  );
  
  create table if not exists kubeflow.observation_logs
  (
      trial_name  varchar(255) not null,
      id          int auto_increment
          primary key,
      time        datetime(6)  null,
      metric_name varchar(255) not null,
      value       text         not null
  );
  
  create table if not exists mlpipeline.pipelines
  (
      UUID             varchar(255)           not null
          primary key,
      CreatedAtInSec   bigint                 not null,
      Name             varchar(255)           not null,
      Description      longtext               not null,
      Parameters       longtext               not null,
      Status           varchar(255)           not null,
      DefaultVersionId varchar(255)           null,
      Namespace        varchar(63) default '' null,
      constraint name_namespace_index
          unique (Name, Namespace),
      constraint namespace_name
          unique (Name, Namespace)
  );
  
  create table if not exists mlpipeline.pipeline_versions
  (
      UUID            varchar(255) not null
          primary key,
      CreatedAtInSec  bigint       not null,
      Name            varchar(255) not null,
      Parameters      longtext     not null,
      PipelineId      varchar(255) not null,
      Status          varchar(255) not null,
      CodeSourceUrl   varchar(255) null,
      Description     longtext     not null,
      PipelineSpec    longtext     not null,
      PipelineSpecURI longtext     not null,
      constraint idx_pipelineid_name
          unique (Name, PipelineId),
      constraint pipeline_versions_PipelineId_pipelines_UUID_foreign
          foreign key (PipelineId) references mlpipeline.pipelines (UUID)
              on update cascade on delete cascade
  );
  
  create index idx_pipeline_versions_CreatedAtInSec
      on mlpipeline.pipeline_versions (CreatedAtInSec);
  
  create index idx_pipeline_versions_PipelineId
      on mlpipeline.pipeline_versions (PipelineId);
  
  create table if not exists mlpipeline.resource_references
  (
      ResourceUUID  varchar(255) not null,
      ResourceType  varchar(255) not null,
      ReferenceUUID varchar(255) not null,
      ReferenceName varchar(255) not null,
      ReferenceType varchar(255) not null,
      Relationship  varchar(255) not null,
      Payload       longtext     not null,
      primary key (ResourceUUID, ResourceType, ReferenceType)
  );
  
  create index referencefilter
      on mlpipeline.resource_references (ResourceType, ReferenceUUID, ReferenceType);
  
  create table if not exists mlpipeline.run_details
  (
      UUID                    varchar(255)     not null
          primary key,
      ExperimentUUID          varchar(255)     not null,
      DisplayName             varchar(255)     not null,
      Name                    varchar(255)     not null,
      StorageState            varchar(255)     not null,
      Namespace               varchar(255)     not null,
      ServiceAccount          varchar(255)     not null,
      Description             varchar(255)     not null,
      CreatedAtInSec          bigint           not null,
      ScheduledAtInSec        bigint default 0 null,
      FinishedAtInSec         bigint default 0 null,
      Conditions              varchar(255)     not null,
      PipelineId              varchar(255)     not null,
      PipelineName            varchar(255)     not null,
      PipelineSpecManifest    longtext         null,
      WorkflowSpecManifest    longtext         not null,
      Parameters              longtext         null,
      RuntimeParameters       longtext         null,
      PipelineRoot            longtext         null,
      PipelineRuntimeManifest longtext         not null,
      WorkflowRuntimeManifest longtext         not null,
      JobUUID                 varchar(255)     null,
      PipelineVersionId       varchar(255)     null,
      State                   varchar(255)     null,
      StateHistory            longtext         null,
      PipelineContextId       bigint default 0 null,
      PipelineRunContextId    bigint default 0 null
  );
  
  create index experimentuuid_conditions_finishedatinsec
      on mlpipeline.run_details (ExperimentUUID, Conditions, FinishedAtInSec);
  
  create index experimentuuid_createatinsec
      on mlpipeline.run_details (ExperimentUUID, CreatedAtInSec);
  
  create index namespace_conditions_finishedatinsec
      on mlpipeline.run_details (Namespace, Conditions, FinishedAtInSec);
  
  create index namespace_createatinsec
      on mlpipeline.run_details (Namespace, CreatedAtInSec);
  
  create table if not exists mlpipeline.run_metrics
  (
      RunUUID     varchar(255) not null,
      NodeID      varchar(255) not null,
      Name        varchar(255) not null,
      NumberValue double       null,
      Format      varchar(255) null,
      Payload     longtext     not null,
      primary key (RunUUID, NodeID, Name),
      constraint run_metrics_RunUUID_run_details_UUID_foreign
          foreign key (RunUUID) references mlpipeline.run_details (UUID)
              on update cascade on delete cascade
  );
  
  create table if not exists mlpipeline.tasks
  (
      UUID              varchar(255)     not null
          primary key,
      Namespace         varchar(255)     not null,
      PipelineName      varchar(255)     not null,
      RunUUID           varchar(255)     not null,
      MLMDExecutionID   varchar(255)     not null,
      CreatedTimestamp  bigint           not null,
      FinishedTimestamp bigint           null,
      Fingerprint       varchar(255)     not null,
      PodName           varchar(255)     not null,
      StartedTimestamp  bigint default 0 null,
      Name              varchar(255)     null,
      ParentTaskUUID    varchar(255)     null,
      State             varchar(255)     null,
      StateHistory      longtext         null,
      MLMDInputs        longtext         null,
      MLMDOutputs       longtext         null,
      ChildrenPods      longtext         null,
      Payload           longtext         null,
      constraint tasks_RunUUID_run_details_UUID_foreign
          foreign key (RunUUID) references mlpipeline.run_details (UUID)
              on update cascade on delete cascade
  );
  EOT

• Setup the Database and Tables used by mellerikat (Kubeflow, AIConductor, Edge Conductor).

  # path : where kubeflow-table.sql is located
  mysql -h ${DB_HOST} -P ${DB_PORT} -u ${DB_ADMIN_USERNAME} -p < kubeflow-table.sql
  # insert : ${DB_ADMIN_PASSWORD}

---

3. Install Kubeflow

• Set the {variables}.

  export KUBEFLOW_MANIFEST_GIT_URL=https://github.com/mellerikat/kubeflow-on-aws-manifests.git
  export KUBEFLOW_RELEASE_VERSION=v1.8-branch
  
  export KUBEFLOW_INSTALL_VERSION=kf${KUBEFLOW_VERSION_NUM}-eks${AWS_CLUSTER_VERSION_NUM}
  export KUBEFLOW_KATIB_IMAGE=${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/ecr-repo-${AWS_DEFAULT_REGION_ALIAS}-${INFRA_NAME}-${DEPLOY_ENV}/kubeflowkatib/katib-db-manager
  export KUBEFLOW_OIDC_IMAGE=${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/ecr-repo-${AWS_DEFAULT_REGION_ALIAS}-${INFRA_NAME}-${DEPLOY_ENV}/oidc-authservice
  export KUBEFLOW_API_SERVER_IMAGE=${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/ecr-repo-${AWS_DEFAULT_REGION_ALIAS}-${INFRA_NAME}-${DEPLOY_ENV}/ml-pipeline/api-server
  export KUBEFLOW_CACHE_SERVER_IMAGE=${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/ecr-repo-${AWS_DEFAULT_REGION_ALIAS}-${INFRA_NAME}-${DEPLOY_ENV}/ml-pipeline/cache-server

• Proceed with the installation from the pre-created Docker Container.

• Clone the kubeflow on aws manifest.

  # path : ${TOP}
  
  git clone ${KUBEFLOW_MANIFEST_GIT_URL} -b ${KUBEFLOW_INSTALL_VERSION}
  cd kubeflow-on-aws-manifests
  git clone https://github.com/kubeflow/manifests.git upstream  -b ${KUBEFLOW_RELEASE_VERSION}

• Install the necessary tools.

  # path : ${TOP}/kubeflow-on-aws-manifests
  
  # make install-tools
  make install-kustomize install-yq install-python
  
  # NOTE: If you have other versions of python installed then make sure the default is set to python3.8
  alias python=python3.8

• Before proceeding with the installation, execute the following commands to ensure the configured environment variables are correctly input into the installation files.

  # path : ${TOP}/kubeflow-on-aws-manifests
  yq e -i '.metadata.annotations."eks.amazonaws.com/role-arn" = "arn:aws:iam::'${AWS_ACCOUNT_ID}':role/role-'${INFRA_NAME}'-'${DEPLOY_ENV}'-kubeflow-secrets-manager-sa"' ai-conductor-configs/common/aws-secrets-manager/kubeflow-secrets-manager-sa.yaml
  
  yq e -i '.spec.parameters.objects |= sub("/parameter.*s3",env(AWS_SECRETS_MANAGER_S3))' awsconfigs/common/aws-secrets-manager/s3/secret-provider.yaml
  yq e -i '.spec.parameters.objects |= sub("/parameter.*rds",env(AWS_SECRETS_MANAGER_RDS))' awsconfigs/common/aws-secrets-manager/rds/secret-provider.yaml
  
  directories=(awsconfigs/apps/pipeline/s3 awsconfigs/apps/pipeline-static/s3)
  
  # The content to be written
  content=$(printf '
  bucketName=%s
  minioServiceHost=%s
  minioServiceRegion=%s
  ' "s3-${AWS_DEFAULT_REGION_ALIAS}-${INFRA_NAME}-${DEPLOY_ENV}-kubeflow" "s3.amazonaws.com" "${AWS_DEFAULT_REGION}")
  
  # Loop through the directories and write the content to params.env
  for dir in "${directories[@]}"; do
    echo "$content" > "$dir/params.env"
  done
  
  sed -i "s|host.*|host: '${DB_HOST}'|g" ai-conductor-configs/apps/pipeline-static/server_config
  sed -i "s|port.*|port: ${DB_PORT}|g" ai-conductor-configs/apps/pipeline-static/server_config
  sed -i "s|user.*|user: '${DB_APP_USERNAME}'|g" ai-conductor-configs/apps/pipeline-static/server_config
  sed -i "s|password.*|password: '${DB_APP_PASSWORD}'|g" ai-conductor-configs/apps/pipeline-static/server_config
  
  sed -i "s|region:.*|region: '${AWS_DEFAULT_REGION}',|g" awsconfigs/apps/pipeline-static/s3/config
  sed -i "s|region:.*|region: '${AWS_DEFAULT_REGION}',|g" ai-conductor-configs/apps/pipeline-static/workflow-controller-config
  
  yq e -i '.images[].newName = env(KUBEFLOW_KATIB_IMAGE)' ai-conductor-configs/apps/katib/kustomization.yaml
  yq e -i '.images[].newTag = env(KUBEFLOW_INSTALL_VERSION)' ai-conductor-configs/apps/katib/kustomization.yaml
  
  yq e -i '.images[].newName = env(KUBEFLOW_OIDC_IMAGE)' ai-conductor-configs/common/oidc-authservice/kustomization.yaml
  yq e -i '.images[].newTag = env(KUBEFLOW_INSTALL_VERSION)' ai-conductor-configs/common/oidc-authservice/kustomization.yaml
  
  sed -i "s|SESSION_STORE_REDIS_ADDR=.*|SESSION_STORE_REDIS_ADDR=${REDIS_HOST}:${REDIS_PORT}|g" ai-conductor-configs/common/oidc-authservice/params.env
  
  yq e -i '.images[0].newName = env(KUBEFLOW_API_SERVER_IMAGE)' ai-conductor-configs/apps/pipeline-static/kustomization.yaml
  yq e -i '.images[0].newTag = env(KUBEFLOW_INSTALL_VERSION)' ai-conductor-configs/apps/pipeline-static/kustomization.yaml
  
  yq e -i '.images[1].newName = env(KUBEFLOW_CACHE_SERVER_IMAGE)' ai-conductor-configs/apps/pipeline-static/kustomization.yaml
  yq e -i '.images[1].newTag = env(KUBEFLOW_INSTALL_VERSION)' ai-conductor-configs/apps/pipeline-static/kustomization.yaml
  
  kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/secrets-store-csi-driver/v1.3.2/deploy/rbac-secretproviderclass.yaml
  kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/secrets-store-csi-driver/v1.3.2/deploy/csidriver.yaml
  kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/secrets-store-csi-driver/v1.3.2/deploy/secrets-store.csi.x-k8s.io_secretproviderclasses.yaml
  kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/secrets-store-csi-driver/v1.3.2/deploy/secrets-store.csi.x-k8s.io_secretproviderclasspodstatuses.yaml
  kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/secrets-store-csi-driver/v1.3.2/deploy/secrets-store-csi-driver.yaml
  kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/secrets-store-csi-driver/v1.3.2/deploy/rbac-secretprovidersyncing.yaml
  kubectl apply -f https://raw.githubusercontent.com/aws/secrets-store-csi-driver-provider-aws/main/deployment/aws-provider-installer.yaml
  
  printf '
  dbHost='$DB_HOST'
  mlmdDb='metadb'
  ' > awsconfigs/apps/pipeline/rds/params.env
  
  printf '
  dbPort='$DB_PORT'
  mysqlPort='$DB_PORT'
  mysqlHost='$DB_HOST'
  ' > ai-conductor-configs/apps/pipeline-static/params.env

• Install the required python packages.

  pip install -r tests/e2e/requirements.txt

• Start the Kubeflow installation.

  # Set variables for Makefile.aiconductor
  export CLUSTER_NAME=${AWS_CLUSTER_NAME}
  export CLUSTER_REGION=${AWS_DEFAULT_REGION}
  
  make -f Makefile.aiconductor deploy-ai-conductor INSTALLATION_OPTION=kustomize DEPLOYMENT_OPTION=rds-s3 PIPELINE_S3_CREDENTIAL_OPTION=static

• Verify that you have Kubeflow installed.

  • NOTE : The kubeflow dashboard can be accessed after AI Conductor Deployment.

  kubectl get pod -n kubeflow
  [Expand Verify Kubeflow installed]

  Once the installation is complete, the pod's STATUS will be in the 'Running' state, as shown below.

  NAME                                                    READY   STATUS    RESTARTS       AGE
  admission-webhook-deployment-6d5d56594c-zq8rm           1/1     Running   0              15s
  aws-secrets-sync-6f5c68bc86-4n8g7                       2/2     Running   0              15s
  cache-server-5d594c7fd-mj7ph                            2/2     Running   0              15s
  centraldashboard-7fbdfffd44-hcx68                       2/2     Running   0              15s
  jupyter-web-app-deployment-66f5df6dcb-x66d8             2/2     Running   0              15s
  katib-controller-dc948b86-mcmw6                         1/1     Running   0              15s
  katib-db-manager-6b5c46597b-z9k5p                       1/1     Running   0              15s
  katib-ui-644b544659-7tcb8                               2/2     Running   0              15s
  kserve-controller-manager-5887b6d64-5dlgv               2/2     Running   0              15s
  kserve-models-web-app-669d78b778-dthwm                  2/2     Running   0              15s
  kubeflow-pipelines-profile-controller-fd55c5b8f-vb4z8   1/1     Running   0              15s
  metacontroller-0                                        1/1     Running   0              15s
  metadata-envoy-deployment-df9cc8756-wgcpg               1/1     Running   0              15s
  metadata-grpc-deployment-84584b656f-rf7w2               2/2     Running   0              15s
  metadata-writer-8d848b6ff-h746k                         2/2     Running   0              15s
  ml-pipeline-59dbd488f8-ft4p8                            2/2     Running   0              15s
  ml-pipeline-persistenceagent-7cc6dcd89b-xvkpp           2/2     Running   0              15s
  ml-pipeline-scheduledworkflow-6d88c5665d-gjrz7          2/2     Running   0              15s
  ml-pipeline-ui-774489c54b-q24ml                         2/2     Running   0              15s
  ml-pipeline-viewer-crd-5f45cd49c-7wkhm                  2/2     Running   0              15s
  ml-pipeline-visualizationserver-6744c57c9d-pshbn        2/2     Running   0              15s
  notebook-controller-deployment-78668b8d48-x8zkz         2/2     Running   0              15s
  profiles-deployment-6b88ffb46c-8mvlc                    3/3     Running   0              15s
  tensorboard-controller-deployment-74c8b8d44f-8tl4r      3/3     Running   0              15s
  tensorboards-web-app-deployment-5646975d68-8sppp        2/2     Running   0              15s
  training-operator-5b78f8948b-66sqt                      1/1     Running   0              15s
  volumes-web-app-deployment-7c4db4d478-ndc4q             2/2     Running   0              15s
  workflow-controller-656cffb8b-c4q6n                     2/2     Running   0              15s

• Uninstall Kubeflow

  [Expand Delete Kubeflow]

  # Delete Kubeflow
  make -f Makefile.aiconductor delete-kubeflow