{ "cells": [ { "cell_type": "markdown", "id": "4848d63e-8844-444e-adba-205200244138", "metadata": {}, "source": [ "# SKA-Mid conversion guide" ] }, { "cell_type": "code", "execution_count": 1, "id": "14a699d5-bf6e-4ebe-bfa5-dcccbec0ec2e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "XRADIO version 1.1.3 already installed.\n" ] } ], "source": [ "from importlib.metadata import version\n", "import os\n", "\n", "try:\n", " import xradio\n", "\n", " print(\"XRADIO version\", version(\"xradio\"), \"already installed.\")\n", "except ImportError as e:\n", " print(e)\n", " print(\"Installing XRADIO\")\n", "\n", " os.system(\"pip install xradio\")\n", "\n", " import xradio\n", "\n", " print(\"xradio version\", version(\"xradio\"), \" installed.\")" ] }, { "cell_type": "markdown", "id": "a939f97e-6c76-47b0-aef5-256df2e908f9", "metadata": {}, "source": [ "## Download dataset" ] }, { "cell_type": "code", "execution_count": 2, "id": "134a99e3-c5a2-443c-96d4-f44fae59555a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[\u001b[38;2;128;05;128m2026-04-20 15:20:21,456\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m toolviper: \u001b[0m Initializing download... \n", "[\u001b[38;2;128;05;128m2026-04-20 15:20:21,457\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m toolviper: \u001b[0m File already exists: /Users/vdesouza/work/xradio/docs/source/measurement_set/guides/AA2-Mid-sim_00000.ms \n" ] } ], "source": [ "import toolviper\n", "\n", "toolviper.utils.data.download(file=\"AA2-Mid-sim_00000.ms\")" ] }, { "cell_type": "markdown", "id": "f8435c96-7010-4b79-8be6-51f7f6993b5f", "metadata": {}, "source": [ "## Convert to Processing Set" ] }, { "cell_type": "code", "execution_count": 3, "id": "ee52d124-2c17-450b-879f-1f86f0ae265c", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[\u001b[38;2;128;05;128m2026-04-20 15:20:21,755\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m toolviper: \u001b[0m Updated partition scheme used: ['DATA_DESC_ID', 'OBSERVATION_ID'] \n", "[\u001b[38;2;128;05;128m2026-04-20 15:20:21,756\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m toolviper: \u001b[0m Number of partitions: 1 \n", "[\u001b[38;2;128;05;128m2026-04-20 15:20:21,756\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m toolviper: \u001b[0m OBSERVATION_ID [0], DDI [0], STATE [None], FIELD [0], SCAN [0], EPHEMERIS [None] \n", "[\u001b[38;2;128;05;128m2026-04-20 15:20:21,898\u001b[0m] \u001b[38;2;255;160;0m WARNING\u001b[0m\u001b[38;2;112;128;144m toolviper: \u001b[0m SOURCE table empty for (unique) source_id [0] and spectral_window_id 0. \n" ] } ], "source": [ "from xradio.measurement_set import convert_msv2_to_processing_set\n", "\n", "ms_file = \"AA2-Mid-sim_00000.ms\"\n", "\n", "main_chunksize = {\"frequency\": 1, \"time\": 20} # baseline, polarization\n", "outfile = \"AA2-Mid-sim_00000.ps.zarr\"\n", "convert_msv2_to_processing_set(\n", " in_file=ms_file,\n", " out_file=outfile,\n", " parallel_mode=\"none\",\n", " persistence_mode='w',\n", " main_chunksize=main_chunksize,\n", ")" ] }, { "cell_type": "markdown", "id": "fbd02679-0df8-4fa5-8036-6f22f534e386", "metadata": {}, "source": [ "## Processing Set" ] }, { "cell_type": "code", "execution_count": 4, "id": "dab986ca-55f8-4a4a-ba59-9c97fcc2ca84", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
namescan_intentsshapeexecution_block_UIDpolarizationscan_namespw_namespw_intentsfield_namesource_nameline_namefield_coordssession_reference_UIDscheduling_block_UIDproject_UIDstart_frequencyend_frequency
0AA2-Mid-sim_00000_0[None](95, 1891, 3, 4)---[XX, XY, YX, YY][0]spw_0[UNSPECIFIED][_0][Unknown][][fk5, 0h00m00.00s, -45d00m00.00s]------950000000.09.500989e+08
\n", "
" ], "text/plain": [ " name scan_intents shape execution_block_UID \\\n", "0 AA2-Mid-sim_00000_0 [None] (95, 1891, 3, 4) --- \n", "\n", " polarization scan_name spw_name spw_intents field_name source_name \\\n", "0 [XX, XY, YX, YY] [0] spw_0 [UNSPECIFIED] [_0] [Unknown] \n", "\n", " line_name field_coords session_reference_UID \\\n", "0 [] [fk5, 0h00m00.00s, -45d00m00.00s] --- \n", "\n", " scheduling_block_UID project_UID start_frequency end_frequency \n", "0 --- 950000000.0 9.500989e+08 " ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "ps_name = outfile\n", "\n", "from xradio.measurement_set import open_processing_set\n", "\n", "ps_xdt = open_processing_set(ps_store=outfile)\n", "ps_xdt.xr_ps.summary()" ] }, { "cell_type": "code", "execution_count": 5, "id": "29367c0c-50eb-4e45-b69f-1492b54e4a15", "metadata": {}, "outputs": [], "source": [ "ms_xdt = ps_xdt[\"AA2-Mid-sim_00000_0\"]" ] }, { "cell_type": "code", "execution_count": 6, "id": "e7168ef8-dc7b-4114-8beb-a18910d19b5e", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "
<xarray.DatasetView> Size: 16kB\n",
       "Dimensions:                       (field_name: 1, sky_dir_label: 2,\n",
       "                                   baseline_id: 1891, frequency: 3,\n",
       "                                   polarization: 4, time: 95, uvw_label: 3)\n",
       "Coordinates:\n",
       "  * field_name                    (field_name) <U23 92B '_0'\n",
       "    source_name                   (field_name) <U7 28B dask.array<chunksize=(1,), meta=np.ndarray>\n",
       "  * sky_dir_label                 (sky_dir_label) <U3 24B 'ra' 'dec'\n",
       "  * baseline_id                   (baseline_id) int64 15kB 0 1 2 ... 1889 1890\n",
       "  * frequency                     (frequency) float64 24B 9.5e+08 ... 9.501e+08\n",
       "  * polarization                  (polarization) <U2 32B 'XX' 'XY' 'YX' 'YY'\n",
       "  * time                          (time) float64 760B 9.467e+08 ... 9.467e+08\n",
       "  * uvw_label                     (uvw_label) <U1 12B 'u' 'v' 'w'\n",
       "Data variables:\n",
       "    FIELD_PHASE_CENTER_DIRECTION  (field_name, sky_dir_label) float64 16B dask.array<chunksize=(1, 2), meta=np.ndarray>\n",
       "Attributes:\n",
       "    type:     field_and_source
" ], "text/plain": [ " Size: 16kB\n", "Dimensions: (field_name: 1, sky_dir_label: 2,\n", " baseline_id: 1891, frequency: 3,\n", " polarization: 4, time: 95, uvw_label: 3)\n", "Coordinates:\n", " * field_name (field_name) \n", " * sky_dir_label (sky_dir_label) \n", "Attributes:\n", " type: field_and_source" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "ms_xdt.xr_ms.get_field_and_source_xds()" ] }, { "cell_type": "code", "execution_count": 7, "id": "ba761deb-b7a9-4b2b-8163-3050413fe59c", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "
<xarray.DataTree 'antenna_xds'>\n",
       "Group: /AA2-Mid-sim_00000_0/antenna_xds\n",
       "    Dimensions:                 (time: 95, baseline_id: 1891, frequency: 3,\n",
       "                                 polarization: 4, uvw_label: 3, antenna_name: 62,\n",
       "                                 cartesian_pos_label: 3, receptor_label: 2)\n",
       "    Coordinates:\n",
       "      * antenna_name            (antenna_name) <U5 1kB 's0000' 's0001' ... 's0061'\n",
       "        mount                   (antenna_name) <U5 1kB dask.array<chunksize=(62,), meta=np.ndarray>\n",
       "        station_name            (antenna_name) <U5 1kB dask.array<chunksize=(62,), meta=np.ndarray>\n",
       "        telescope_name          (antenna_name) <U35 9kB dask.array<chunksize=(62,), meta=np.ndarray>\n",
       "      * cartesian_pos_label     (cartesian_pos_label) <U1 12B 'x' 'y' 'z'\n",
       "      * receptor_label          (receptor_label) <U5 40B 'pol_0' 'pol_1'\n",
       "        polarization_type       (antenna_name, receptor_label) <U1 496B dask.array<chunksize=(62, 2), meta=np.ndarray>\n",
       "    Inherited coordinates:\n",
       "      * baseline_id             (baseline_id) int64 15kB 0 1 2 3 ... 1888 1889 1890\n",
       "      * frequency               (frequency) float64 24B 9.5e+08 9.5e+08 9.501e+08\n",
       "      * polarization            (polarization) <U2 32B 'XX' 'XY' 'YX' 'YY'\n",
       "      * time                    (time) float64 760B 9.467e+08 ... 9.467e+08\n",
       "      * uvw_label               (uvw_label) <U1 12B 'u' 'v' 'w'\n",
       "    Data variables:\n",
       "        ANTENNA_DISH_DIAMETER   (antenna_name) float64 496B dask.array<chunksize=(62,), meta=np.ndarray>\n",
       "        ANTENNA_POSITION        (antenna_name, cartesian_pos_label) float64 1kB dask.array<chunksize=(62, 3), meta=np.ndarray>\n",
       "        ANTENNA_RECEPTOR_ANGLE  (antenna_name, receptor_label) float64 992B dask.array<chunksize=(62, 2), meta=np.ndarray>\n",
       "    Attributes:\n",
       "        overall_telescope_name:  OSKAR 2.8.4-dev 2023-08-22 d1b86f8c\n",
       "        relocatable_antennas:    False\n",
       "        type:                    antenna
" ], "text/plain": [ "\n", "Group: /AA2-Mid-sim_00000_0/antenna_xds\n", " Dimensions: (time: 95, baseline_id: 1891, frequency: 3,\n", " polarization: 4, uvw_label: 3, antenna_name: 62,\n", " cartesian_pos_label: 3, receptor_label: 2)\n", " Coordinates:\n", " * antenna_name (antenna_name) \n", " station_name (antenna_name) \n", " telescope_name (antenna_name) \n", " * cartesian_pos_label (cartesian_pos_label) \n", " Inherited coordinates:\n", " * baseline_id (baseline_id) int64 15kB 0 1 2 3 ... 1888 1889 1890\n", " * frequency (frequency) float64 24B 9.5e+08 9.5e+08 9.501e+08\n", " * polarization (polarization) \n", " ANTENNA_POSITION (antenna_name, cartesian_pos_label) float64 1kB dask.array\n", " ANTENNA_RECEPTOR_ANGLE (antenna_name, receptor_label) float64 992B dask.array\n", " Attributes:\n", " overall_telescope_name: OSKAR 2.8.4-dev 2023-08-22 d1b86f8c\n", " relocatable_antennas: False\n", " type: antenna" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "ms_xdt.antenna_xds" ] }, { "cell_type": "code", "execution_count": 8, "id": "d4237165-4a77-4665-a799-be358a737d02", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "
<xarray.DatasetView> Size: 35MB\n",
       "Dimensions:                     (time: 95, baseline_id: 1891, frequency: 3,\n",
       "                                 polarization: 4, uvw_label: 3)\n",
       "Coordinates:\n",
       "  * time                        (time) float64 760B 9.467e+08 ... 9.467e+08\n",
       "    field_name                  (time) <U23 9kB dask.array<chunksize=(95,), meta=np.ndarray>\n",
       "    scan_name                   (time) <U21 8kB dask.array<chunksize=(95,), meta=np.ndarray>\n",
       "  * baseline_id                 (baseline_id) int64 15kB 0 1 2 ... 1889 1890\n",
       "    baseline_antenna1_name      (baseline_id) <U5 38kB dask.array<chunksize=(1891,), meta=np.ndarray>\n",
       "    baseline_antenna2_name      (baseline_id) <U5 38kB dask.array<chunksize=(1891,), meta=np.ndarray>\n",
       "  * frequency                   (frequency) float64 24B 9.5e+08 ... 9.501e+08\n",
       "  * polarization                (polarization) <U2 32B 'XX' 'XY' 'YX' 'YY'\n",
       "  * uvw_label                   (uvw_label) <U1 12B 'u' 'v' 'w'\n",
       "Data variables:\n",
       "    EFFECTIVE_INTEGRATION_TIME  (time, baseline_id) float64 1MB dask.array<chunksize=(20, 1891), meta=np.ndarray>\n",
       "    FLAG                        (time, baseline_id, frequency, polarization) bool 2MB dask.array<chunksize=(20, 1891, 1, 4), meta=np.ndarray>\n",
       "    TIME_CENTROID               (time, baseline_id) float64 1MB dask.array<chunksize=(20, 1891), meta=np.ndarray>\n",
       "    UVW                         (time, baseline_id, uvw_label) float64 4MB dask.array<chunksize=(20, 1891, 3), meta=np.ndarray>\n",
       "    VISIBILITY                  (time, baseline_id, frequency, polarization) complex64 17MB dask.array<chunksize=(20, 1891, 1, 4), meta=np.ndarray>\n",
       "    WEIGHT                      (time, baseline_id, frequency, polarization) float32 9MB dask.array<chunksize=(20, 1891, 1, 4), meta=np.ndarray>\n",
       "Attributes:\n",
       "    creation_date:     2026-04-20T21:20:21.785491+00:00\n",
       "    creator:           {'software_name': 'xradio', 'version': '1.1.3'}\n",
       "    data_groups:       {'base': {'correlated_data': 'VISIBILITY', 'date': '20...\n",
       "    observation_info:  {'observer': ['fred'], 'observing_log': "['']", 'proje...\n",
       "    processor_info:    {'sub_type': '', 'type': ''}\n",
       "    schema_version:    4.0.0\n",
       "    type:              visibility
" ], "text/plain": [ " Size: 35MB\n", "Dimensions: (time: 95, baseline_id: 1891, frequency: 3,\n", " polarization: 4, uvw_label: 3)\n", "Coordinates:\n", " * time (time) float64 760B 9.467e+08 ... 9.467e+08\n", " field_name (time) \n", " scan_name (time) \n", " * baseline_id (baseline_id) int64 15kB 0 1 2 ... 1889 1890\n", " baseline_antenna1_name (baseline_id) \n", " baseline_antenna2_name (baseline_id) \n", " * frequency (frequency) float64 24B 9.5e+08 ... 9.501e+08\n", " * polarization (polarization) \n", " FLAG (time, baseline_id, frequency, polarization) bool 2MB dask.array\n", " TIME_CENTROID (time, baseline_id) float64 1MB dask.array\n", " UVW (time, baseline_id, uvw_label) float64 4MB dask.array\n", " VISIBILITY (time, baseline_id, frequency, polarization) complex64 17MB dask.array\n", " WEIGHT (time, baseline_id, frequency, polarization) float32 9MB dask.array\n", "Attributes:\n", " creation_date: 2026-04-20T21:20:21.785491+00:00\n", " creator: {'software_name': 'xradio', 'version': '1.1.3'}\n", " data_groups: {'base': {'correlated_data': 'VISIBILITY', 'date': '20...\n", " observation_info: {'observer': ['fred'], 'observing_log': \"['']\", 'proje...\n", " processor_info: {'sub_type': '', 'type': ''}\n", " schema_version: 4.0.0\n", " type: visibility" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "ms_xdt.ds" ] }, { "cell_type": "code", "execution_count": 9, "id": "74120772-9830-4da5-b8e2-f4ab42de1f54", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "
<xarray.Dataset> Size: 32kB\n",
       "Dimensions:                 (antenna_name: 62, cartesian_pos_label: 3,\n",
       "                             receptor_label: 2, baseline_id: 1891,\n",
       "                             frequency: 3, polarization: 4, time: 95,\n",
       "                             uvw_label: 3)\n",
       "Coordinates:\n",
       "  * antenna_name            (antenna_name) <U5 1kB 's0000' 's0001' ... 's0061'\n",
       "    mount                   (antenna_name) <U5 1kB 'FIXED' 'FIXED' ... 'FIXED'\n",
       "    station_name            (antenna_name) <U5 1kB 's0000' 's0001' ... 's0061'\n",
       "    telescope_name          (antenna_name) <U35 9kB 'OSKAR 2.8.4-dev 2023-08-...\n",
       "  * cartesian_pos_label     (cartesian_pos_label) <U1 12B 'x' 'y' 'z'\n",
       "  * receptor_label          (receptor_label) <U5 40B 'pol_0' 'pol_1'\n",
       "    polarization_type       (antenna_name, receptor_label) <U1 496B 'X' ... 'Y'\n",
       "  * baseline_id             (baseline_id) int64 15kB 0 1 2 3 ... 1888 1889 1890\n",
       "  * frequency               (frequency) float64 24B 9.5e+08 9.5e+08 9.501e+08\n",
       "  * polarization            (polarization) <U2 32B 'XX' 'XY' 'YX' 'YY'\n",
       "  * time                    (time) float64 760B 9.467e+08 ... 9.467e+08\n",
       "  * uvw_label               (uvw_label) <U1 12B 'u' 'v' 'w'\n",
       "Data variables:\n",
       "    ANTENNA_DISH_DIAMETER   (antenna_name) float64 496B 1.0 1.0 1.0 ... 1.0 1.0\n",
       "    ANTENNA_POSITION        (antenna_name, cartesian_pos_label) float64 1kB 5...\n",
       "    ANTENNA_RECEPTOR_ANGLE  (antenna_name, receptor_label) float64 992B 0.0 ....\n",
       "Attributes:\n",
       "    overall_telescope_name:  OSKAR 2.8.4-dev 2023-08-22 d1b86f8c\n",
       "    relocatable_antennas:    False\n",
       "    type:                    antenna
" ], "text/plain": [ " Size: 32kB\n", "Dimensions: (antenna_name: 62, cartesian_pos_label: 3,\n", " receptor_label: 2, baseline_id: 1891,\n", " frequency: 3, polarization: 4, time: 95,\n", " uvw_label: 3)\n", "Coordinates:\n", " * antenna_name (antenna_name) " ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "ps_xdt.xr_ps.plot_antenna_positions_2d()" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.13.13" }, "widgets": { "application/vnd.jupyter.widget-state+json": { "state": {}, "version_major": 2, "version_minor": 0 } } }, "nbformat": 4, "nbformat_minor": 5 }