Mlflow

Confusion matrix for cross validation. Results are being saved as mlflow artifacts and retrieved later for evaluation purposes. import mlflow import numpy as np from sklearn.ensemble import RandomForestClassifier from sklearn.metrics import precision_recall_fscore_support from sklearn.metrics import accuracy_score from sklearn.metrics import confusion_matrix from sklearn.datasets import load_breast_cancer from sklearn.model_selection import StratifiedKFold SEED = 42 mlflow.set_tracking_uri("http://127.0.0.1:5000") mlflow.set_experiment("cross-validation-average") # Data preparation dataset = load_breast_cancer() X = dataset.data y = dataset.target print(X.shape, y.shape) # Model training skf = StratifiedKFold(n_splits=5, random_state=SEED, shuffle=True) for train_index, test_index in skf.split(X, y): X_train, X_test = X[train_index], X[test_index] y_train, y_test = y[train_index], y[test_index] with mlflow.start_run(): mlflow.sklearn.autolog(exclusive=False) model = RandomForestClassifier(random_state=SEED) model.fit(X_train, y_train) y_pred = model.predict(X_test) mlflow.log_dict({"y_test": [int(x) for x in y_test], "y_pred": [int(x) for x in y_pred] }, "ytest-ypred.json") test_acc = accuracy_score(y_test, y_pred) mlflow.log_metric("test_accuracy", test_acc) print("test_accuracy:", test_acc) test_precision, test_recall, test_f1, _ = precision_recall_fscore_support( y_test, y_pred, average='binary' ) mlflow.log_metric("test_precision", test_precision) mlflow.log_metric("test_recall", test_recall) mlflow.log_metric("test_f1_score", test_f1) tn, fp, fn, tp = confusion_matrix(y_test, y_pred).ravel() mlflow.log_metric("tn", tn) mlflow.log_metric("fp", fp) mlflow.log_metric("fn", fn) mlflow.log_metric("tp", tp) tn, fp, fn, tp = confusion_matrix( y_test, y_pred, normalize="true" ).ravel() mlflow.log_metric("tn_normalized", tn) mlflow.log_metric("fp_normalized", fp) mlflow.log_metric("fn_normalized", fn) mlflow.log_metric("tp_normalized", tp) mlflow.sklearn.autolog(disable=True) # Results evaluation runs = mlflow.search_runs(experiment_ids=["2"]) columns = [ 'metrics.tn_normalized', 'metrics.fp_normalized', 'metrics.fn_normalized', 'metrics.tp_normalized' ] mean_confusion_matrix = runs[columns].mean() print(mean_confusion_matrix) Output: ...

How to save mlflow runs and artifacts into an external folder Install the mlflow lib: pip install mlflow Create a folder artifacts: mkdir mlflow-artifacts Create a folder sqlite database(s): mkdir mlflow-dbs Start the mlflow UI in terminal: mlflow ui \ --backend-store-uri sqlite:///mlflow-dbs/db-20220822.sqlite \ --default-artifact-root mlflow-artifacts/ The mlflow UI will be served on http://127.0.0.1:5000 Create a project folder: your-project Switch to the project directory: cd your-project Connect to the UI and run experiments: # experiment.py import mlflow mlflow.set_tracking_uri("http://127.0.0.1:5000") mlflow.set_experiment("experiment-001") with mlflow.start_run(): mlflow.log_param("num_layers", 5) mlflow.log_metric("accuracy", 0.75) Run the script: python experiment.py Autologging contents to an active fluent run, which may be user-created: ...

MLFlow Quickstart import mlflow from sklearn.tree import DecisionTreeClassifier from sklearn.metrics import accuracy_score, balanced_accuracy_score, f1_score EXPERIMENT_NAME = 'mlflow-experiment' EXPERIMENT_ID = mlflow.create_experiment(EXPERIMENT_NAME) X_train, y_train, X_test, y_test = ... depth = ... model = DecisionTreeClassifier(max_depth=depth) model.fit(X_train, y_train) y_pred = model.predict(X_test) acc = accuracy_score(y_test, y_pred) balanced_acc = balanced_accuracy_score(y_true, y_pred) f1 = f1_score(y_true) RUN_NAME = 'run-1' with mlflow.start_run(experiment_id=EXPERIMENT_ID, run_name=RUN_NAME) as run: # Track parameters mlflow.log_param('depth', depth) # Track metrics mlflow.log_metric('accuracy', accuracy) mlflow.log_metrics({'balanced_acc': balanced_acc, 'f1': f1}) # Track model mlflow.sklearn.log_model(model, 'dt-classifier') # Launch the MLFlow Web UI, accessible at http://localhost:5000 # !mlflow ui # Retrieve experiment/run results prorammatically and compare them client = mlflow.tracking.MlflowClient() # Retrieve Experiment information EXPERIMENT_ID = client.get_experiment_by_name(EXPERIMENT_NAME).experiment_id # Retrieve Runs information (parameter 'depth', metric 'accuracy', 'balanced_accuracy', 'f1-score') ALL_RUNS_INFO = client.list_run_infos(EXPERIMENT_ID) ALL_RUNS_ID = [run.run_id for run in ALL_RUNS_INFO] ALL_PARAM = [client.get_run(run_id).data.params['depth'] for run_id in ALL_RUNS_ID] ALL_METRIC = [client.get_run(run_id).data.metrics['accuracy'] for run_id in ALL_RUNS_ID] # View Runs information df = pd.DataFrame({'run_id': ALL_RUNS_ID, 'params': ALL_PARAM, 'metrics': ALL_METRIC}) # Retrieve Artifact from best run best_run_id = df.sort_values('metrics', ascending=False).iloc[0]['run_id'] best_model_path = client.download_artifacts(best_run_id, 'dt-classifier') best_model = mlflow.sklearn.load_model(best_model_path)