{ "cells": [ { "cell_type": "code", "execution_count": null, "id": "2a409dd5", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "WARNING:tensorflow:From d:\\Anaconda\\Lib\\site-packages\\tf_keras\\src\\losses.py:2976: The name tf.losses.sparse_softmax_cross_entropy is deprecated. Please use tf.compat.v1.losses.sparse_softmax_cross_entropy instead.\n", "\n", "Menggunakan perangkat: cuda\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[I 2025-07-18 06:26:20,055] A new study created in memory with name: no-name-50af0249-7af4-476f-988c-7342adeab58c\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Memulai hyperparameter tuning dengan Optuna...\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Some weights of BertForTokenClassification were not initialized from the model checkpoint at indobenchmark/indobert-base-p1 and are newly initialized: ['classifier.bias', 'classifier.weight']\n", "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n", "C:\\Users\\BUDI\\AppData\\Local\\Temp\\ipykernel_6152\\2584540621.py:147: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n", " trainer = Trainer(\n" ] }, { "data": { "text/html": [ "\n", "
\n", " \n", " \n", " [836/836 03:00, Epoch 4/4]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation LossPrecisionRecallF1AccuracyPer Entity
10.1247000.1668680.7480680.7311180.7394960.945582{}
20.1038000.1578930.7503550.7990940.7739580.952456{}
30.0961000.1719320.8006130.7885200.7945210.955606{}
40.0328000.1786150.7507040.8051360.7769680.954031{}

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "Trainer is attempting to log a value of \"{}\" of type for key \"eval_per_entity\" as a metric. MLflow's log_metric() only accepts float and int types so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval/per_entity\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval_per_entity\" as a metric. MLflow's log_metric() only accepts float and int types so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval/per_entity\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval_per_entity\" as a metric. MLflow's log_metric() only accepts float and int types so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval/per_entity\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval_per_entity\" as a metric. MLflow's log_metric() only accepts float and int types so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval/per_entity\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n" ] }, { "data": { "text/html": [ "\n", "

\n", " \n", " \n", " [27/27 00:01]\n", "
\n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "Trainer is attempting to log a value of \"{}\" of type for key \"eval_per_entity\" as a metric. MLflow's log_metric() only accepts float and int types so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval/per_entity\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n", "[I 2025-07-18 06:29:29,091] Trial 0 finished with value: 0.7945205479452055 and parameters: {'learning_rate': 2.3555847899573657e-05, 'batch_size': 8, 'num_epochs': 4}. Best is trial 0 with value: 0.7945205479452055.\n", "Some weights of BertForTokenClassification were not initialized from the model checkpoint at indobenchmark/indobert-base-p1 and are newly initialized: ['classifier.bias', 'classifier.weight']\n", "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n", "C:\\Users\\BUDI\\AppData\\Local\\Temp\\ipykernel_6152\\2584540621.py:147: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n", " trainer = Trainer(\n" ] }, { "data": { "text/html": [ "\n", "
\n", " \n", " \n", " [1045/1045 04:05, Epoch 5/5]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation LossPrecisionRecallF1AccuracyPer Entity
10.1235000.1634880.7287880.7265860.7276850.945009{}
20.1088000.1556140.7373460.8141990.7738690.953745{}
30.1103000.1704700.7633140.7794560.7713000.953172{}
40.0458000.1823730.7655570.7990940.7819660.954031{}
50.0224000.1911590.7585710.8021150.7797360.953315{}

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "Trainer is attempting to log a value of \"{}\" of type for key \"eval_per_entity\" as a metric. MLflow's log_metric() only accepts float and int types so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval/per_entity\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval_per_entity\" as a metric. MLflow's log_metric() only accepts float and int types so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval/per_entity\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval_per_entity\" as a metric. MLflow's log_metric() only accepts float and int types so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval/per_entity\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval_per_entity\" as a metric. MLflow's log_metric() only accepts float and int types so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval/per_entity\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval_per_entity\" as a metric. MLflow's log_metric() only accepts float and int types so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval/per_entity\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n" ] }, { "data": { "text/html": [ "\n", "

\n", " \n", " \n", " [27/27 00:01]\n", "
\n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "Trainer is attempting to log a value of \"{}\" of type for key \"eval_per_entity\" as a metric. MLflow's log_metric() only accepts float and int types so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval/per_entity\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n", "[I 2025-07-18 06:33:40,086] Trial 1 finished with value: 0.7819660014781965 and parameters: {'learning_rate': 1.7904807706862636e-05, 'batch_size': 8, 'num_epochs': 5}. Best is trial 0 with value: 0.7945205479452055.\n", "Some weights of BertForTokenClassification were not initialized from the model checkpoint at indobenchmark/indobert-base-p1 and are newly initialized: ['classifier.bias', 'classifier.weight']\n", "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n", "C:\\Users\\BUDI\\AppData\\Local\\Temp\\ipykernel_6152\\2584540621.py:147: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n", " trainer = Trainer(\n" ] }, { "data": { "text/html": [ "\n", "
\n", " \n", " \n", " [420/420 05:47, Epoch 4/4]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation LossPrecisionRecallF1AccuracyPer Entity
10.1386000.1855500.7387690.6706950.7030880.942432{}
20.1098000.1546190.7818990.7960730.7889220.955463{}
30.0698000.1550780.8077500.8187310.8132030.960332{}
40.0272000.1742920.7652920.8126890.7882780.954747{}

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "Trainer is attempting to log a value of \"{}\" of type for key \"eval_per_entity\" as a metric. MLflow's log_metric() only accepts float and int types so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval/per_entity\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval_per_entity\" as a metric. MLflow's log_metric() only accepts float and int types so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval/per_entity\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval_per_entity\" as a metric. MLflow's log_metric() only accepts float and int types so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval/per_entity\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval_per_entity\" as a metric. MLflow's log_metric() only accepts float and int types so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval/per_entity\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n" ] }, { "data": { "text/html": [ "\n", "

\n", " \n", " \n", " [14/14 00:00]\n", "
\n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "Trainer is attempting to log a value of \"{}\" of type for key \"eval_per_entity\" as a metric. MLflow's log_metric() only accepts float and int types so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval/per_entity\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n", "[I 2025-07-18 06:39:32,835] Trial 2 finished with value: 0.8132033008252062 and parameters: {'learning_rate': 3.672145523121866e-05, 'batch_size': 16, 'num_epochs': 4}. Best is trial 2 with value: 0.8132033008252062.\n", "Some weights of BertForTokenClassification were not initialized from the model checkpoint at indobenchmark/indobert-base-p1 and are newly initialized: ['classifier.bias', 'classifier.weight']\n", "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n", "C:\\Users\\BUDI\\AppData\\Local\\Temp\\ipykernel_6152\\2584540621.py:147: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n", " trainer = Trainer(\n" ] }, { "data": { "text/html": [ "\n", "
\n", " \n", " \n", " [525/525 07:42, Epoch 5/5]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation LossPrecisionRecallF1AccuracyPer Entity
10.1432000.1709700.7455140.6903320.7168630.945869{}
20.1073000.1544060.7661410.8066470.7858720.953029{}
30.0751000.1585030.7954200.7870090.7911920.956895{}
40.0258000.1793480.7647910.8006040.7822880.954461{}
50.0134000.1852570.7660490.8111780.7879680.953888{}

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "Trainer is attempting to log a value of \"{}\" of type for key \"eval_per_entity\" as a metric. MLflow's log_metric() only accepts float and int types so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval/per_entity\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval_per_entity\" as a metric. MLflow's log_metric() only accepts float and int types so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval/per_entity\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval_per_entity\" as a metric. MLflow's log_metric() only accepts float and int types so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval/per_entity\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval_per_entity\" as a metric. MLflow's log_metric() only accepts float and int types so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval/per_entity\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval_per_entity\" as a metric. MLflow's log_metric() only accepts float and int types so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval/per_entity\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n" ] }, { "data": { "text/html": [ "\n", "

\n", " \n", " \n", " [14/14 00:01]\n", "
\n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "Trainer is attempting to log a value of \"{}\" of type for key \"eval_per_entity\" as a metric. MLflow's log_metric() only accepts float and int types so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval/per_entity\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n", "[I 2025-07-18 06:47:22,280] Trial 3 finished with value: 0.7911921032649962 and parameters: {'learning_rate': 3.713773945286763e-05, 'batch_size': 16, 'num_epochs': 5}. Best is trial 2 with value: 0.8132033008252062.\n", "Some weights of BertForTokenClassification were not initialized from the model checkpoint at indobenchmark/indobert-base-p1 and are newly initialized: ['classifier.bias', 'classifier.weight']\n", "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n", "C:\\Users\\BUDI\\AppData\\Local\\Temp\\ipykernel_6152\\2584540621.py:147: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n", " trainer = Trainer(\n" ] }, { "data": { "text/html": [ "\n", "
\n", " \n", " \n", " [1045/1045 04:30, Epoch 5/5]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation LossPrecisionRecallF1AccuracyPer Entity
10.1327000.1692050.7153610.7175230.7164400.944007{}
20.1200000.1553900.7507000.8096680.7790700.953458{}
30.1366000.1635550.7619740.7930510.7772020.954174{}
40.0679000.1721240.7664760.8081570.7867650.953888{}
50.0352000.1802490.7599430.8081570.7833090.953745{}

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "Trainer is attempting to log a value of \"{}\" of type for key \"eval_per_entity\" as a metric. MLflow's log_metric() only accepts float and int types so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval/per_entity\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval_per_entity\" as a metric. MLflow's log_metric() only accepts float and int types so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval/per_entity\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval_per_entity\" as a metric. MLflow's log_metric() only accepts float and int types so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval/per_entity\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval_per_entity\" as a metric. MLflow's log_metric() only accepts float and int types so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval/per_entity\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval_per_entity\" as a metric. MLflow's log_metric() only accepts float and int types so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval/per_entity\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n" ] }, { "data": { "text/html": [ "\n", "

\n", " \n", " \n", " [27/27 00:01]\n", "
\n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "Trainer is attempting to log a value of \"{}\" of type for key \"eval_per_entity\" as a metric. MLflow's log_metric() only accepts float and int types so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval/per_entity\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n", "[I 2025-07-18 06:51:59,633] Trial 4 finished with value: 0.7867647058823529 and parameters: {'learning_rate': 1.1923156920458335e-05, 'batch_size': 8, 'num_epochs': 5}. Best is trial 2 with value: 0.8132033008252062.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "Hyperparameter terbaik:\n", "{'learning_rate': 3.672145523121866e-05, 'batch_size': 16, 'num_epochs': 4}\n", "F1-Score terbaik: 0.8132\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Some weights of BertForTokenClassification were not initialized from the model checkpoint at indobenchmark/indobert-base-p1 and are newly initialized: ['classifier.bias', 'classifier.weight']\n", "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n", "C:\\Users\\BUDI\\AppData\\Local\\Temp\\ipykernel_6152\\2584540621.py:195: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n", " trainer = Trainer(\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "Memulai pelatihan dengan hyperparameter terbaik...\n" ] }, { "data": { "text/html": [ "\n", "
\n", " \n", " \n", " [420/420 07:01, Epoch 4/4]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation LossPrecisionRecallF1AccuracyPer Entity
10.1386000.1855500.7387690.6706950.7030880.942432{}
20.1098000.1546190.7818990.7960730.7889220.955463{}
30.0698000.1550780.8077500.8187310.8132030.960332{}
40.0272000.1742920.7652920.8126890.7882780.954747{}

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "Trainer is attempting to log a value of \"{}\" of type for key \"eval_per_entity\" as a metric. MLflow's log_metric() only accepts float and int types so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval/per_entity\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval_per_entity\" as a metric. MLflow's log_metric() only accepts float and int types so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval/per_entity\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval_per_entity\" as a metric. MLflow's log_metric() only accepts float and int types so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval/per_entity\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval_per_entity\" as a metric. MLflow's log_metric() only accepts float and int types so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval/per_entity\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "Mengevaluasi model pada data test...\n" ] }, { "data": { "text/html": [ "\n", "

\n", " \n", " \n", " [14/14 00:05]\n", "
\n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "Trainer is attempting to log a value of \"{}\" of type for key \"eval_per_entity\" as a metric. MLflow's log_metric() only accepts float and int types so we dropped this attribute.\n", "Trainer is attempting to log a value of \"{}\" of type for key \"eval/per_entity\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "Hasil Evaluasi:\n", "Precision: 0.7528\n", "Recall: 0.7878\n", "F1-Score: 0.7699\n", "Accuracy: 0.9497\n", "\n", "Metrik per Entitas:\n", "\n", "Model dan tokenizer telah disimpan ke './ner_model'\n", "\n", "Contoh Prediksi pada Data Test (5 Sampel):\n", "\n", "Sampel 1:\n", "Tokens: [CLS] joe ##tat ##a hadi ##hard ##aja dan dihadiri oleh rektor undip prof . [SEP]\n", "True Labels: ['B-PERSON', 'I-PERSON', 'O', 'O', 'O', 'O', 'B-ORGANISATION', 'O', 'O']\n", "Predicted Labels: ['B-PERSON', 'I-PERSON', 'O', 'O', 'O', 'O', 'B-PLACE', 'O', 'O']\n", "\n", "Sampel 2:\n", "Tokens: [CLS] sejak masih duduk di bangku sekolah tk kevin sudah belajar alat musik piano secara formal dan ketika ia menginjak sekolah smp pemilik nama asli kevin april ##io sum ##aat ##maj ##a ini , mulai belajar menulis lagu sendiri . [SEP]\n", "True Labels: ['O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-PERSON', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-PERSON', 'I-PERSON', 'I-PERSON', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']\n", "Predicted Labels: ['O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-PERSON', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-PERSON', 'I-PERSON', 'I-PERSON', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']\n", "\n", "Sampel 3:\n", "Tokens: [CLS] pada tanggal 6 februari 1976 , wakil ketua lock ##he ##ed corporation memberitahu subk ##omi ##te senat as bahwa tana ##ka selaku pm telah dibayar ( dis ##ogo ##k ) sebagai ganjaran pembelian pesawat lock ##he ##ed l - 1011 . [SEP]\n", "True Labels: ['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-ORGANISATION', 'I-ORGANISATION', 'O', 'O', 'O', 'B-PLACE', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-ORGANISATION', 'O', 'O', 'O', 'O']\n", "Predicted Labels: ['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-ORGANISATION', 'I-ORGANISATION', 'O', 'O', 'O', 'B-PLACE', 'O', 'B-PERSON', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-ORGANISATION', 'O', 'O', 'O', 'O']\n", "\n", "Sampel 4:\n", "Tokens: [CLS] dengan kondisi alam yang sejuk dan curah hujan yang tinggi maka didaerah tersebut banyak didapati bermacam jenis flora dan fauna seperti : gajah yang di kenal dengan legenda poc ##ut me ##urah ##nya , rusa , harimau , beruang , kancil , babi hutan , tengg ##iling , landak dan ular , juga terdapat berbagai macam jenis burung yang selalu menghiasi kawasan ini . [SEP]\n", "True Labels: ['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']\n", "Predicted Labels: ['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']\n", "\n", "Sampel 5:\n", "Tokens: [CLS] awak pesawat yang terdiri atas pilot ard ##y ted ##jo , kopi ##lot h ribuan dan dua awak lainnya perry reh ##ata dan mei ##nas ##ta segera membuka pintu pesawat dan menurunkan penumpang dengan selamat . tanggal 14 juni 2009 , hari minggu , pukul 09 . 20 , pesawat terbang express air jenis dor ##nie ##r d ##32 ##8 - 100 bernomor badan pk - tx ##n , mengalami kecelakaan saat mendarat . [SEP]\n", "True Labels: ['O', 'O', 'O', 'O', 'O', 'O', 'B-PERSON', 'I-PERSON', 'O', 'O', 'B-PERSON', 'I-PERSON', 'O', 'O', 'O', 'O', 'B-PERSON', 'I-PERSON', 'O', 'B-PERSON', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-ORGANISATION', 'I-ORGANISATION', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']\n", "Predicted Labels: ['O', 'O', 'O', 'O', 'O', 'O', 'B-PERSON', 'I-PERSON', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-PERSON', 'I-PERSON', 'O', 'B-PERSON', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']\n", "\n", "Analisis Pola Error (Tanggal diprediksi sebagai Lokasi):\n", "Tidak ditemukan contoh tanggal yang diprediksi sebagai lokasi dalam 100 sampel.\n", "\n", "Pertimbangan Keamanan Data, Privasi, dan Etika:\n", "- Dataset bersumber dari berita publik, tidak mengandung informasi sensitif seperti alamat atau nomor identitas.\n", "- Nama orang dalam dataset berasal dari media publik, aman untuk digunakan.\n", "- Dataset mencakup berbagai topik berita, mengurangi risiko bias terhadap entitas tertentu.\n" ] } ], "source": [ "import os\n", "import sys\n", "import subprocess\n", "import numpy as np\n", "from datasets import load_dataset\n", "from transformers import AutoTokenizer, AutoModelForTokenClassification, DataCollatorForTokenClassification, Trainer, TrainingArguments\n", "import evaluate\n", "import torch\n", "import optuna\n", "\n", "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", "print(f\"Menggunakan perangkat: {device}\")\n", "\n", "# Load dataset\n", "try:\n", " dataset = load_dataset(\"indonlp/indonlu\", \"nergrit\", trust_remote_code=True)\n", "except Exception as e:\n", " print(f\"Gagal memuat dataset: {e}\")\n", " sys.exit(1)\n", "\n", "# Verify dataset structure\n", "if \"train\" not in dataset or \"validation\" not in dataset or \"test\" not in dataset:\n", " print(\"Dataset tidak memiliki split train/validation/test yang diharapkan.\")\n", " sys.exit(1)\n", "if \"tokens\" not in dataset[\"train\"].column_names or \"ner_tags\" not in dataset[\"train\"].column_names:\n", " print(\"Dataset tidak memiliki kolom 'tokens' atau 'ner_tags'.\")\n", " sys.exit(1)\n", "\n", "# Define label list\n", "try:\n", " label_list = dataset[\"train\"].features[\"ner_tags\"].feature.names\n", " label2id = {label: i for i, label in enumerate(label_list)}\n", " id2label = {i: label for i, label in enumerate(label_list)}\n", "except Exception as e:\n", " print(f\"Gagal mendapatkan label: {e}\")\n", " sys.exit(1)\n", "\n", "# Load tokenizer\n", "try:\n", " tokenizer = AutoTokenizer.from_pretrained(\"indobenchmark/indobert-base-p1\")\n", "except Exception as e:\n", " print(f\"Gagal memuat tokenizer: {e}\")\n", " sys.exit(1)\n", "\n", "# Tokenize and align labels\n", "def tokenize_and_align_labels(examples):\n", " tokenized_inputs = tokenizer(examples[\"tokens\"], truncation=True, is_split_into_words=True)\n", " labels = []\n", " for i, label in enumerate(examples[\"ner_tags\"]):\n", " word_ids = tokenized_inputs.word_ids(batch_index=i)\n", " previous_word_idx = None\n", " label_ids = []\n", " for word_idx in word_ids:\n", " if word_idx is None:\n", " label_ids.append(-100)\n", " elif word_idx != previous_word_idx:\n", " label_ids.append(label[word_idx])\n", " else:\n", " label_ids.append(-100)\n", " previous_word_idx = word_idx\n", " labels.append(label_ids)\n", " tokenized_inputs[\"labels\"] = labels\n", " return tokenized_inputs\n", "\n", "# Tokenize dataset\n", "try:\n", " tokenized_dataset = dataset.map(tokenize_and_align_labels, batched=True)\n", "except Exception as e:\n", " print(f\"Gagal menokenisasi dataset: {e}\")\n", " sys.exit(1)\n", "\n", "# Data collator\n", "data_collator = DataCollatorForTokenClassification(tokenizer)\n", "\n", "# Load evaluation metric\n", "metric = evaluate.load(\"seqeval\")\n", "\n", "# Compute metrics\n", "def compute_metrics(p):\n", " predictions, labels = p\n", " predictions = np.argmax(predictions, axis=2)\n", " true_labels = [[id2label[l] for l in label if l != -100] for label in labels]\n", " pred_labels = [[id2label[p] for p, l in zip(prediction, label) if l != -100] for prediction, label in zip(predictions, labels)]\n", " results = metric.compute(predictions=pred_labels, references=true_labels)\n", " per_entity = {}\n", " for entity in [\"PERSON\", \"ORGANISATION\", \"PLACE\", \"DATE\"]:\n", " if entity.lower() in results:\n", " per_entity[entity] = {\n", " \"precision\": results[entity.lower()][\"precision\"],\n", " \"recall\": results[entity.lower()][\"recall\"],\n", " \"f1\": results[entity.lower()][\"f1\"],\n", " }\n", " return {\n", " \"precision\": results[\"overall_precision\"],\n", " \"recall\": results[\"overall_recall\"],\n", " \"f1\": results[\"overall_f1\"],\n", " \"accuracy\": results[\"overall_accuracy\"],\n", " \"per_entity\": per_entity,\n", " }\n", "\n", "# Define objective function for Optuna\n", "def objective(trial):\n", " # Define hyperparameter search space\n", " learning_rate = trial.suggest_float(\"learning_rate\", 1e-5, 5e-5, log=True)\n", " batch_size = trial.suggest_categorical(\"batch_size\", [8, 16, 32])\n", " num_epochs = trial.suggest_int(\"num_epochs\", 3, 5)\n", "\n", " # Load model for each trial\n", " model = AutoModelForTokenClassification.from_pretrained(\n", " \"indobenchmark/indobert-base-p1\",\n", " num_labels=len(label_list),\n", " id2label=id2label,\n", " label2id=label2id\n", " )\n", " model.to(device)\n", "\n", " # Set training arguments\n", " training_args = TrainingArguments(\n", " output_dir=f\"./results_trial_{trial.number}\",\n", " eval_strategy=\"epoch\",\n", " learning_rate=learning_rate,\n", " per_device_train_batch_size=batch_size,\n", " per_device_eval_batch_size=batch_size,\n", " num_train_epochs=num_epochs,\n", " weight_decay=0.01,\n", " logging_dir=f\"./logs_trial_{trial.number}\",\n", " logging_steps=10,\n", " save_strategy=\"epoch\",\n", " load_best_model_at_end=True,\n", " metric_for_best_model=\"f1\",\n", " )\n", "\n", " # Initialize Trainer\n", " trainer = Trainer(\n", " model=model,\n", " args=training_args,\n", " train_dataset=tokenized_dataset[\"train\"],\n", " eval_dataset=tokenized_dataset[\"validation\"],\n", " tokenizer=tokenizer,\n", " data_collator=data_collator,\n", " compute_metrics=compute_metrics,\n", " )\n", "\n", " # Train and evaluate\n", " trainer.train()\n", " eval_results = trainer.evaluate()\n", " return eval_results[\"eval_f1\"]\n", "\n", "# Run Optuna optimization\n", "print(\"Memulai hyperparameter tuning dengan Optuna...\")\n", "study = optuna.create_study(direction=\"maximize\")\n", "study.optimize(objective, n_trials=5) # Adjust n_trials as needed\n", "print(\"\\nHyperparameter terbaik:\")\n", "print(study.best_params)\n", "print(f\"F1-Score terbaik: {study.best_value:.4f}\")\n", "\n", "# Train final model with best hyperparameters\n", "best_params = study.best_params\n", "model = AutoModelForTokenClassification.from_pretrained(\n", " \"indobenchmark/indobert-base-p1\",\n", " num_labels=len(label_list),\n", " id2label=id2label,\n", " label2id=label2id\n", ")\n", "model.to(device)\n", "\n", "training_args = TrainingArguments(\n", " output_dir=\"./results\",\n", " eval_strategy=\"epoch\",\n", " learning_rate=best_params[\"learning_rate\"],\n", " per_device_train_batch_size=best_params[\"batch_size\"],\n", " per_device_eval_batch_size=best_params[\"batch_size\"],\n", " num_train_epochs=best_params[\"num_epochs\"],\n", " weight_decay=0.01,\n", " logging_dir=\"./logs\",\n", " logging_steps=10,\n", " save_strategy=\"epoch\",\n", " load_best_model_at_end=True,\n", " metric_for_best_model=\"f1\",\n", ")\n", "\n", "trainer = Trainer(\n", " model=model,\n", " args=training_args,\n", " train_dataset=tokenized_dataset[\"train\"],\n", " eval_dataset=tokenized_dataset[\"validation\"],\n", " tokenizer=tokenizer,\n", " data_collator=data_collator,\n", " compute_metrics=compute_metrics,\n", ")\n", "\n", "# Train the model\n", "print(\"\\nMemulai pelatihan dengan hyperparameter terbaik...\")\n", "try:\n", " trainer.train()\n", "except Exception as e:\n", " print(f\"Gagal melatih model: {e}\")\n", " sys.exit(1)\n", "\n", "# Evaluate on test set\n", "print(\"\\nMengevaluasi model pada data test...\")\n", "try:\n", " results = trainer.evaluate(tokenized_dataset[\"test\"])\n", "except Exception as e:\n", " print(f\"Gagal mengevaluasi model: {e}\")\n", " sys.exit(1)\n", "\n", "# Print evaluation results\n", "print(\"\\nHasil Evaluasi:\")\n", "print(f\"Precision: {results['eval_precision']:.4f}\")\n", "print(f\"Recall: {results['eval_recall']:.4f}\")\n", "print(f\"F1-Score: {results['eval_f1']:.4f}\")\n", "print(f\"Accuracy: {results['eval_accuracy']:.4f}\")\n", "print(\"\\nMetrik per Entitas:\")\n", "for entity, metrics in results.get(\"eval_per_entity\", {}).items():\n", " print(f\"{entity}:\")\n", " print(f\" Precision: {metrics['precision']:.4f}\")\n", " print(f\" Recall: {metrics['recall']:.4f}\")\n", " print(f\" F1-Score: {metrics['f1']:.4f}\")\n", "\n", "# Save the model\n", "try:\n", " model.save_pretrained(\"./ner_model\")\n", " tokenizer.save_pretrained(\"./ner_model\")\n", " print(\"\\nModel dan tokenizer telah disimpan ke './ner_model'\")\n", "except Exception as e:\n", " print(f\"Gagal menyimpan model: {e}\")\n", " sys.exit(1)\n", "\n", "# Example inference on test samples\n", "print(\"\\nContoh Prediksi pada Data Test (5 Sampel):\")\n", "try:\n", " for i in range(min(5, len(tokenized_dataset[\"test\"]))):\n", " sample = tokenized_dataset[\"test\"][i]\n", " input_ids = torch.tensor([sample[\"input_ids\"]], device=device)\n", " attention_mask = torch.tensor([sample[\"attention_mask\"]], device=device)\n", " model.eval()\n", " with torch.no_grad():\n", " outputs = model(input_ids, attention_mask=attention_mask)\n", " predictions = outputs.logits.argmax(dim=2)[0].cpu().numpy()\n", " tokens = tokenizer.convert_ids_to_tokens(sample[\"input_ids\"])\n", " labels = [id2label[pred] for pred, label in zip(predictions, sample[\"labels\"]) if label != -100]\n", " true_labels = [id2label[label] for label in sample[\"labels\"] if label != -100]\n", " print(f\"\\nSampel {i+1}:\")\n", " print(f\"Tokens: {' '.join(tokens)}\")\n", " print(f\"True Labels: {true_labels}\")\n", " print(f\"Predicted Labels: {labels}\")\n", "except Exception as e:\n", " print(f\"Gagal melakukan inferensi: {e}\")\n", " sys.exit(1)\n", "\n", "# Analyze error patterns (DATE predicted as LOC)\n", "print(\"\\nAnalisis Pola Error (Tanggal diprediksi sebagai Lokasi):\")\n", "found_error = False\n", "for i in range(min(100, len(tokenized_dataset[\"test\"]))):\n", " sample = tokenized_dataset[\"test\"][i]\n", " input_ids = torch.tensor([sample[\"input_ids\"]], device=device)\n", " attention_mask = torch.tensor([sample[\"attention_mask\"]], device=device)\n", " with torch.no_grad():\n", " outputs = model(input_ids, attention_mask=attention_mask)\n", " predictions = outputs.logits.argmax(dim=2)[0].cpu().numpy()\n", " true_labels = [id2label[label] for label in sample[\"labels\"] if label != -100]\n", " pred_labels = [id2label[pred] for pred, label in zip(predictions, sample[\"labels\"]) if label != -100]\n", " for j, (true, pred) in enumerate(zip(true_labels, pred_labels)):\n", " if true.startswith(\"B-DATE\") and pred.startswith(\"B-LOC\"):\n", " tokens = tokenizer.convert_ids_to_tokens(sample[\"input_ids\"])\n", " print(f\"\\nSampel dengan Error (DATE diprediksi sebagai LOC):\")\n", " print(f\"Tokens: {' '.join(tokens)}\")\n", " print(f\"True Labels: {true_labels}\")\n", " print(f\"Predicted Labels: {pred_labels}\")\n", " found_error = True\n", " break\n", " if found_error:\n", " break\n", "if not found_error:\n", " print(\"Tidak ditemukan contoh tanggal yang diprediksi sebagai lokasi dalam 100 sampel.\")\n", "\n", "# Data Security, Privacy, and Ethics\n", "print(\"\\nPertimbangan Keamanan Data, Privasi, dan Etika:\")\n", "print(\"- Dataset bersumber dari berita publik, tidak mengandung informasi sensitif seperti alamat atau nomor identitas.\")\n", "print(\"- Nama orang dalam dataset berasal dari media publik, aman untuk digunakan.\")\n", "print(\"- Dataset mencakup berbagai topik berita, mengurangi risiko bias terhadap entitas tertentu.\")" ] }, { "cell_type": "code", "execution_count": null, "id": "714cfb72", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "93508875", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "base", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.7" } }, "nbformat": 4, "nbformat_minor": 5 }