{ "cells": [ { "cell_type": "markdown", "id": "c8db6f12-c072-49ef-96e8-6392e46511ff", "metadata": {}, "source": [ "# Global VLBI conversion guide" ] }, { "cell_type": "code", "execution_count": 1, "id": "14a699d5-bf6e-4ebe-bfa5-dcccbec0ec2e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "XRADIO version 1.1.3 already installed.\n" ] } ], "source": [ "from importlib.metadata import version\n", "import os\n", "\n", "try:\n", " import xradio\n", "\n", " print(\"XRADIO version\", version(\"xradio\"), \"already installed.\")\n", "except ImportError as e:\n", " print(e)\n", " print(\"Installing XRADIO\")\n", "\n", " os.system(\"pip install xradio\")\n", "\n", " import xradio\n", "\n", " print(\"xradio version\", version(\"xradio\"), \" installed.\")" ] }, { "cell_type": "markdown", "id": "a939f97e-6c76-47b0-aef5-256df2e908f9", "metadata": {}, "source": [ "## Download dataset" ] }, { "cell_type": "markdown", "id": "4e478428", "metadata": {}, "source": [ "global (EVN+VLBA) VLBI observation with:\n", "- 5 scans \n", "- 2 fields \n", "- 2 spw" ] }, { "cell_type": "code", "execution_count": 2, "id": "134a99e3-c5a2-443c-96d4-f44fae59555a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[\u001b[38;2;128;05;128m2026-04-20 15:21:21,727\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m toolviper: \u001b[0m Initializing download... \n", "[\u001b[38;2;128;05;128m2026-04-20 15:21:21,727\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m toolviper: \u001b[0m File already exists: /Users/vdesouza/work/xradio/docs/source/measurement_set/guides/global_vlbi_gg084b_reduced.ms \n" ] } ], "source": [ "import toolviper\n", "toolviper.utils.data.download(\"global_vlbi_gg084b_reduced.ms\")" ] }, { "cell_type": "markdown", "id": "f8435c96-7010-4b79-8be6-51f7f6993b5f", "metadata": {}, "source": [ "## Convert to Processing Set" ] }, { "cell_type": "code", "execution_count": 3, "id": "ee52d124-2c17-450b-879f-1f86f0ae265c", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[\u001b[38;2;128;05;128m2026-04-20 15:21:22,206\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m toolviper: \u001b[0m Updated partition scheme used: ['DATA_DESC_ID', 'OBSERVATION_ID'] \n", "[\u001b[38;2;128;05;128m2026-04-20 15:21:22,207\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m toolviper: \u001b[0m Number of partitions: 2 \n", "[\u001b[38;2;128;05;128m2026-04-20 15:21:22,207\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m toolviper: \u001b[0m OBSERVATION_ID [0], DDI [0], STATE [None], FIELD [0, 1], SCAN [3, 319, 320, 321, 552], EPHEMERIS [None] \n", "[\u001b[38;2;128;05;128m2026-04-20 15:21:22,399\u001b[0m] \u001b[38;2;255;160;0m WARNING\u001b[0m\u001b[38;2;112;128;144m toolviper: \u001b[0m Source_id is -1. No source information will be included in the field_and_source_xds. \n", "[\u001b[38;2;128;05;128m2026-04-20 15:21:22,801\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m toolviper: \u001b[0m OBSERVATION_ID [0], DDI [1], STATE [None], FIELD [0, 1], SCAN [3, 319, 320, 321, 552], EPHEMERIS [None] \n", "[\u001b[38;2;128;05;128m2026-04-20 15:21:22,981\u001b[0m] \u001b[38;2;255;160;0m WARNING\u001b[0m\u001b[38;2;112;128;144m toolviper: \u001b[0m Source_id is -1. No source information will be included in the field_and_source_xds. \n" ] } ], "source": [ "from xradio.measurement_set import convert_msv2_to_processing_set\n", "\n", "ms_file = \"global_vlbi_gg084b_reduced.ms\"\n", "\n", "main_chunksize = {\"frequency\": 1, \"time\": 20} # baseline, polarization\n", "outfile = \"global_vlbi_gg084b_reduced.ps.zarr\"\n", "convert_msv2_to_processing_set(\n", " in_file=ms_file,\n", " out_file=outfile,\n", " parallel_mode=\"none\",\n", " persistence_mode='w',\n", " main_chunksize=main_chunksize,\n", ")" ] }, { "cell_type": "markdown", "id": "fbd02679-0df8-4fa5-8036-6f22f534e386", "metadata": {}, "source": [ "## Processing Set" ] }, { "cell_type": "code", "execution_count": 4, "id": "dab986ca-55f8-4a4a-ba59-9c97fcc2ca84", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
| \n", " | name | \n", "scan_intents | \n", "shape | \n", "execution_block_UID | \n", "polarization | \n", "scan_name | \n", "spw_name | \n", "spw_intents | \n", "field_name | \n", "source_name | \n", "line_name | \n", "field_coords | \n", "session_reference_UID | \n", "scheduling_block_UID | \n", "project_UID | \n", "start_frequency | \n", "end_frequency | \n", "
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | \n", "global_vlbi_gg084b_reduced_0 | \n", "[None] | \n", "(480, 57, 32, 4) | \n", "--- | \n", "[RR, RL, LR, LL] | \n", "[3, 319, 320, 321, 552] | \n", "spw_0 | \n", "[UNSPECIFIED] | \n", "[EM170817_1, J1311-2329_0] | \n", "[Unknown] | \n", "[] | \n", "Multi-Phase-Center | \n", "--- | \n", "--- | \n", "GG084B | \n", "4.900500e+09 | \n", "4.916000e+09 | \n", "
| 1 | \n", "global_vlbi_gg084b_reduced_1 | \n", "[None] | \n", "(480, 57, 32, 4) | \n", "--- | \n", "[RR, RL, LR, LL] | \n", "[3, 319, 320, 321, 552] | \n", "spw_1 | \n", "[UNSPECIFIED] | \n", "[EM170817_1, J1311-2329_0] | \n", "[Unknown] | \n", "[] | \n", "Multi-Phase-Center | \n", "--- | \n", "--- | \n", "GG084B | \n", "4.916000e+09 | \n", "4.931500e+09 | \n", "
<xarray.DataTree 'global_vlbi_gg084b_reduced_0'>\n",
"Group: /global_vlbi_gg084b_reduced_0\n",
"│ Dimensions: (time: 480, baseline_id: 57, frequency: 32,\n",
"│ polarization: 4, uvw_label: 3)\n",
"│ Coordinates:\n",
"│ * time (time) float64 4kB 1.527e+09 ... 1.527e+09\n",
"│ field_name (time) <U32 61kB dask.array<chunksize=(480,), meta=np.ndarray>\n",
"│ scan_name (time) <U21 40kB dask.array<chunksize=(480,), meta=np.ndarray>\n",
"│ * baseline_id (baseline_id) int64 456B 0 1 2 3 ... 53 54 55 56\n",
"│ baseline_antenna1_name (baseline_id) <U2 456B dask.array<chunksize=(57,), meta=np.ndarray>\n",
"│ baseline_antenna2_name (baseline_id) <U2 456B dask.array<chunksize=(57,), meta=np.ndarray>\n",
"│ * frequency (frequency) float64 256B 4.9e+09 ... 4.916e+09\n",
"│ * polarization (polarization) <U2 32B 'RR' 'RL' 'LR' 'LL'\n",
"│ * uvw_label (uvw_label) <U1 12B 'u' 'v' 'w'\n",
"│ Data variables:\n",
"│ EFFECTIVE_INTEGRATION_TIME (time, baseline_id) float64 219kB dask.array<chunksize=(20, 57), meta=np.ndarray>\n",
"│ FLAG (time, baseline_id, frequency, polarization) bool 4MB dask.array<chunksize=(20, 57, 1, 4), meta=np.ndarray>\n",
"│ TIME_CENTROID (time, baseline_id) float64 219kB dask.array<chunksize=(20, 57), meta=np.ndarray>\n",
"│ UVW (time, baseline_id, uvw_label) float64 657kB dask.array<chunksize=(20, 57, 3), meta=np.ndarray>\n",
"│ VISIBILITY (time, baseline_id, frequency, polarization) complex64 28MB dask.array<chunksize=(20, 57, 1, 4), meta=np.ndarray>\n",
"│ WEIGHT (time, baseline_id, frequency, polarization) float32 14MB dask.array<chunksize=(20, 57, 1, 4), meta=np.ndarray>\n",
"│ Attributes:\n",
"│ creation_date: 2026-04-20T21:21:22.222037+00:00\n",
"│ creator: {'software_name': 'xradio', 'version': '1.1.3'}\n",
"│ data_groups: {'base': {'correlated_data': 'VISIBILITY', 'date': '20...\n",
"│ observation_info: {'observer': ['GG084B'], 'observing_log': '[]', 'proje...\n",
"│ processor_info: {'sub_type': '', 'type': ''}\n",
"│ schema_version: 4.0.0\n",
"│ type: visibility\n",
"├── Group: /global_vlbi_gg084b_reduced_0/antenna_xds\n",
"│ Dimensions: (antenna_name: 14, cartesian_pos_label: 3,\n",
"│ receptor_label: 2)\n",
"│ Coordinates:\n",
"│ * antenna_name (antenna_name) <U2 112B 'BD' 'HH' 'YS' ... 'BR' 'MK'\n",
"│ mount (antenna_name) <U16 896B dask.array<chunksize=(14,), meta=np.ndarray>\n",
"│ station_name (antenna_name) <U2 112B dask.array<chunksize=(14,), meta=np.ndarray>\n",
"│ telescope_name (antenna_name) <U3 168B dask.array<chunksize=(14,), meta=np.ndarray>\n",
"│ * cartesian_pos_label (cartesian_pos_label) <U1 12B 'x' 'y' 'z'\n",
"│ * receptor_label (receptor_label) <U5 40B 'pol_0' 'pol_1'\n",
"│ polarization_type (antenna_name, receptor_label) <U1 112B dask.array<chunksize=(14, 2), meta=np.ndarray>\n",
"│ Data variables:\n",
"│ ANTENNA_DISH_DIAMETER (antenna_name) float64 112B dask.array<chunksize=(14,), meta=np.ndarray>\n",
"│ ANTENNA_POSITION (antenna_name, cartesian_pos_label) float64 336B dask.array<chunksize=(14, 3), meta=np.ndarray>\n",
"│ ANTENNA_RECEPTOR_ANGLE (antenna_name, receptor_label) float64 224B dask.array<chunksize=(14, 2), meta=np.ndarray>\n",
"│ Attributes:\n",
"│ overall_telescope_name: EVN\n",
"│ relocatable_antennas: False\n",
"│ type: antenna\n",
"├── Group: /global_vlbi_gg084b_reduced_0/field_and_source_base_xds\n",
"│ Dimensions: (field_name: 2, sky_dir_label: 2)\n",
"│ Coordinates:\n",
"│ * field_name (field_name) <U32 256B 'J1311-2329_0' 'EM17...\n",
"│ source_name (field_name) <U7 56B dask.array<chunksize=(2,), meta=np.ndarray>\n",
"│ * sky_dir_label (sky_dir_label) <U3 24B 'ra' 'dec'\n",
"│ Data variables:\n",
"│ FIELD_PHASE_CENTER_DIRECTION (field_name, sky_dir_label) float64 32B dask.array<chunksize=(2, 2), meta=np.ndarray>\n",
"│ Attributes:\n",
"│ type: field_and_source\n",
"├── Group: /global_vlbi_gg084b_reduced_0/gain_curve_xds\n",
"│ Dimensions: (antenna_name: 14, poly_term: 3, receptor_label: 2)\n",
"│ Coordinates:\n",
"│ * antenna_name (antenna_name) <U2 112B 'BD' 'HH' 'YS' ... 'BR' 'MK'\n",
"│ antenna_id (antenna_name) int32 56B dask.array<chunksize=(14,), meta=np.ndarray>\n",
"│ gain_curve_type (antenna_name) <U9 504B dask.array<chunksize=(14,), meta=np.ndarray>\n",
"│ mount (antenna_name) <U16 896B dask.array<chunksize=(14,), meta=np.ndarray>\n",
"│ station_name (antenna_name) <U2 112B dask.array<chunksize=(14,), meta=np.ndarray>\n",
"│ telescope_name (antenna_name) <U3 168B dask.array<chunksize=(14,), meta=np.ndarray>\n",
"│ * receptor_label (receptor_label) <U5 40B 'pol_0' 'pol_1'\n",
"│ polarization_type (antenna_name, receptor_label) <U1 112B dask.array<chunksize=(14, 2), meta=np.ndarray>\n",
"│ Dimensions without coordinates: poly_term\n",
"│ Data variables:\n",
"│ GAIN_CURVE (antenna_name, poly_term, receptor_label) float64 672B dask.array<chunksize=(14, 3, 2), meta=np.ndarray>\n",
"│ GAIN_CURVE_INTERVAL (antenna_name) float64 112B dask.array<chunksize=(14,), meta=np.ndarray>\n",
"│ GAIN_CURVE_SENSITIVITY (antenna_name, receptor_label) float64 224B dask.array<chunksize=(14, 2), meta=np.ndarray>\n",
"│ Attributes:\n",
"│ measured_date: 2018-05-26T15:33:49.500000000\n",
"│ type: gain_curve\n",
"└── Group: /global_vlbi_gg084b_reduced_0/system_calibration_xds\n",
" Dimensions: (antenna_name: 14, time_system_cal: 10357,\n",
" receptor_label: 2)\n",
" Coordinates:\n",
" * antenna_name (antenna_name) <U2 112B 'BD' 'HH' 'YS' ... 'KP' 'BR' 'MK'\n",
" antenna_id (antenna_name) int32 56B dask.array<chunksize=(14,), meta=np.ndarray>\n",
" mount (antenna_name) <U16 896B dask.array<chunksize=(14,), meta=np.ndarray>\n",
" station_name (antenna_name) <U2 112B dask.array<chunksize=(14,), meta=np.ndarray>\n",
" telescope_name (antenna_name) <U3 168B dask.array<chunksize=(14,), meta=np.ndarray>\n",
" * time_system_cal (time_system_cal) float64 83kB 1.527e+09 ... 1.527e+09\n",
" * receptor_label (receptor_label) <U5 40B 'pol_0' 'pol_1'\n",
" polarization_type (antenna_name, receptor_label) <U1 112B dask.array<chunksize=(14, 2), meta=np.ndarray>\n",
" Data variables:\n",
" TSYS (antenna_name, time_system_cal, receptor_label) float64 2MB dask.array<chunksize=(7, 5179, 1), meta=np.ndarray>\n",
" Attributes:\n",
" type: system_calibration<xarray.DataTree 'antenna_xds'>\n",
"Group: /global_vlbi_gg084b_reduced_0/antenna_xds\n",
" Dimensions: (time: 480, baseline_id: 57, frequency: 32,\n",
" polarization: 4, uvw_label: 3, antenna_name: 14,\n",
" cartesian_pos_label: 3, receptor_label: 2)\n",
" Coordinates:\n",
" * antenna_name (antenna_name) <U2 112B 'BD' 'HH' 'YS' ... 'BR' 'MK'\n",
" mount (antenna_name) <U16 896B dask.array<chunksize=(14,), meta=np.ndarray>\n",
" station_name (antenna_name) <U2 112B dask.array<chunksize=(14,), meta=np.ndarray>\n",
" telescope_name (antenna_name) <U3 168B dask.array<chunksize=(14,), meta=np.ndarray>\n",
" * cartesian_pos_label (cartesian_pos_label) <U1 12B 'x' 'y' 'z'\n",
" * receptor_label (receptor_label) <U5 40B 'pol_0' 'pol_1'\n",
" polarization_type (antenna_name, receptor_label) <U1 112B dask.array<chunksize=(14, 2), meta=np.ndarray>\n",
" Inherited coordinates:\n",
" * baseline_id (baseline_id) int64 456B 0 1 2 3 4 ... 53 54 55 56\n",
" * frequency (frequency) float64 256B 4.9e+09 ... 4.916e+09\n",
" * polarization (polarization) <U2 32B 'RR' 'RL' 'LR' 'LL'\n",
" * time (time) float64 4kB 1.527e+09 1.527e+09 ... 1.527e+09\n",
" * uvw_label (uvw_label) <U1 12B 'u' 'v' 'w'\n",
" Data variables:\n",
" ANTENNA_DISH_DIAMETER (antenna_name) float64 112B dask.array<chunksize=(14,), meta=np.ndarray>\n",
" ANTENNA_POSITION (antenna_name, cartesian_pos_label) float64 336B dask.array<chunksize=(14, 3), meta=np.ndarray>\n",
" ANTENNA_RECEPTOR_ANGLE (antenna_name, receptor_label) float64 224B dask.array<chunksize=(14, 2), meta=np.ndarray>\n",
" Attributes:\n",
" overall_telescope_name: EVN\n",
" relocatable_antennas: False\n",
" type: antenna<xarray.DatasetView> Size: 5kB\n",
"Dimensions: (field_name: 2, sky_dir_label: 2,\n",
" baseline_id: 57, frequency: 32,\n",
" polarization: 4, time: 480, uvw_label: 3)\n",
"Coordinates:\n",
" * field_name (field_name) <U32 256B 'J1311-2329_0' 'EM17...\n",
" source_name (field_name) <U7 56B dask.array<chunksize=(2,), meta=np.ndarray>\n",
" * sky_dir_label (sky_dir_label) <U3 24B 'ra' 'dec'\n",
" * baseline_id (baseline_id) int64 456B 0 1 2 3 ... 54 55 56\n",
" * frequency (frequency) float64 256B 4.9e+09 ... 4.916e+09\n",
" * polarization (polarization) <U2 32B 'RR' 'RL' 'LR' 'LL'\n",
" * time (time) float64 4kB 1.527e+09 ... 1.527e+09\n",
" * uvw_label (uvw_label) <U1 12B 'u' 'v' 'w'\n",
"Data variables:\n",
" FIELD_PHASE_CENTER_DIRECTION (field_name, sky_dir_label) float64 32B dask.array<chunksize=(2, 2), meta=np.ndarray>\n",
"Attributes:\n",
" type: field_and_source"
],
"text/plain": [
"<xarray.DataTree 'gain_curve_xds'>\n",
"Group: /global_vlbi_gg084b_reduced_0/gain_curve_xds\n",
" Dimensions: (time: 480, baseline_id: 57, frequency: 32,\n",
" polarization: 4, uvw_label: 3, antenna_name: 14,\n",
" poly_term: 3, receptor_label: 2)\n",
" Coordinates:\n",
" * antenna_name (antenna_name) <U2 112B 'BD' 'HH' 'YS' ... 'BR' 'MK'\n",
" antenna_id (antenna_name) int32 56B dask.array<chunksize=(14,), meta=np.ndarray>\n",
" gain_curve_type (antenna_name) <U9 504B dask.array<chunksize=(14,), meta=np.ndarray>\n",
" mount (antenna_name) <U16 896B dask.array<chunksize=(14,), meta=np.ndarray>\n",
" station_name (antenna_name) <U2 112B dask.array<chunksize=(14,), meta=np.ndarray>\n",
" telescope_name (antenna_name) <U3 168B dask.array<chunksize=(14,), meta=np.ndarray>\n",
" * receptor_label (receptor_label) <U5 40B 'pol_0' 'pol_1'\n",
" polarization_type (antenna_name, receptor_label) <U1 112B dask.array<chunksize=(14, 2), meta=np.ndarray>\n",
" Inherited coordinates:\n",
" * baseline_id (baseline_id) int64 456B 0 1 2 3 4 ... 53 54 55 56\n",
" * frequency (frequency) float64 256B 4.9e+09 ... 4.916e+09\n",
" * polarization (polarization) <U2 32B 'RR' 'RL' 'LR' 'LL'\n",
" * time (time) float64 4kB 1.527e+09 1.527e+09 ... 1.527e+09\n",
" * uvw_label (uvw_label) <U1 12B 'u' 'v' 'w'\n",
" Dimensions without coordinates: poly_term\n",
" Data variables:\n",
" GAIN_CURVE (antenna_name, poly_term, receptor_label) float64 672B dask.array<chunksize=(14, 3, 2), meta=np.ndarray>\n",
" GAIN_CURVE_INTERVAL (antenna_name) float64 112B dask.array<chunksize=(14,), meta=np.ndarray>\n",
" GAIN_CURVE_SENSITIVITY (antenna_name, receptor_label) float64 224B dask.array<chunksize=(14, 2), meta=np.ndarray>\n",
" Attributes:\n",
" measured_date: 2018-05-26T15:33:49.500000000\n",
" type: gain_curve<xarray.DataTree 'system_calibration_xds'>\n",
"Group: /global_vlbi_gg084b_reduced_0/system_calibration_xds\n",
" Dimensions: (time: 480, baseline_id: 57, frequency: 32,\n",
" polarization: 4, uvw_label: 3, antenna_name: 14,\n",
" time_system_cal: 10357, receptor_label: 2)\n",
" Coordinates:\n",
" * antenna_name (antenna_name) <U2 112B 'BD' 'HH' 'YS' ... 'KP' 'BR' 'MK'\n",
" antenna_id (antenna_name) int32 56B dask.array<chunksize=(14,), meta=np.ndarray>\n",
" mount (antenna_name) <U16 896B dask.array<chunksize=(14,), meta=np.ndarray>\n",
" station_name (antenna_name) <U2 112B dask.array<chunksize=(14,), meta=np.ndarray>\n",
" telescope_name (antenna_name) <U3 168B dask.array<chunksize=(14,), meta=np.ndarray>\n",
" * time_system_cal (time_system_cal) float64 83kB 1.527e+09 ... 1.527e+09\n",
" * receptor_label (receptor_label) <U5 40B 'pol_0' 'pol_1'\n",
" polarization_type (antenna_name, receptor_label) <U1 112B dask.array<chunksize=(14, 2), meta=np.ndarray>\n",
" Inherited coordinates:\n",
" * baseline_id (baseline_id) int64 456B 0 1 2 3 4 5 ... 52 53 54 55 56\n",
" * frequency (frequency) float64 256B 4.9e+09 4.901e+09 ... 4.916e+09\n",
" * polarization (polarization) <U2 32B 'RR' 'RL' 'LR' 'LL'\n",
" * time (time) float64 4kB 1.527e+09 1.527e+09 ... 1.527e+09\n",
" * uvw_label (uvw_label) <U1 12B 'u' 'v' 'w'\n",
" Data variables:\n",
" TSYS (antenna_name, time_system_cal, receptor_label) float64 2MB dask.array<chunksize=(7, 5179, 1), meta=np.ndarray>\n",
" Attributes:\n",
" type: system_calibrationglobal_vlbi_gg084b_reduc ... ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100% 0:00:00\n\n", "text/plain": "global_vlbi_gg084b_reduc ... \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m100%\u001b[0m \u001b[36m0:00:00\u001b[0m\n" }, "metadata": {}, "output_type": "display_data" } ] } }, "e11328d243d34065a643067a1f613064": { "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "model_name": "LayoutModel", "state": {} } }, "version_major": 2, "version_minor": 0 } } }, "nbformat": 4, "nbformat_minor": 5 }