Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
"affectedNone": "No runs matching selected criteria.",
"allRuns": "All Runs",
"backwards": "Run Backwards",
"completed": "Completed",
"dateRange": "Date Range",
"errorStartDateBeforeEndDate": "Start Date must be before the End Date",
"exceptionReason": {
Expand All @@ -24,10 +25,12 @@
"permissionDenied": "Dry Run Failed: User does not have permission to create backfills.",
"reprocessBehavior": "Reprocess Behavior",
"run": "Run Backfill",
"running": "Running",
"scheduleNotBackfillable": "This Dag's schedule does not support backfills",
"schedulerPriorityHint": "Backfill Dag runs are ordered after non-backfill Dag runs in each scheduler cycle. Backfill runs may remain queued longer if other non-backfill runs are present.",
"selectDescription": "Run this Dag for a range of dates",
"selectLabel": "Backfill",
"status": "Status",
"title": "Run Backfill",
"toaster": {
"success": {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
},
"favoriteState": "Favorite state",
"favoriteStatePlaceholder": "Select favorite state…",
"hasImportErrors": "Has import errors",
"noTimetableTypesFound": "No timetable types found",
"paused": {
"active": "Not paused",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@ const FILTER_KEYS: Array<FilterableSearchParamsKeys> = [
SearchParamsKeys.ASSET_EVENT_DATE_RANGE,
SearchParamsKeys.DAG_ID,
SearchParamsKeys.TASK_ID,
SearchParamsKeys.SOURCE_RUN_ID,
SearchParamsKeys.SOURCE_MAP_INDEX,
SearchParamsKeys.PARTITION_KEY,
];

export const AssetEventsFilter = () => {
Expand Down
37 changes: 37 additions & 0 deletions airflow-core/src/airflow/ui/src/constants/filterConfigs.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import {
MdCode,
MdComputer,
MdDateRange,
MdErrorOutline,
MdHistory,
MdHourglassEmpty,
MdLoop,
Expand Down Expand Up @@ -114,6 +115,15 @@ export const useFilterConfigs = () => {
}));

const filterConfigMap = {
[SearchParamsKeys.ACTIVE]: {
icon: <MdPlayArrow />,
label: translate("components:backfill.status"),
options: [
{ label: translate("components:backfill.running"), value: "true" },
{ label: translate("components:backfill.completed"), value: "false" },
],
type: FilterTypes.SELECT,
},
[SearchParamsKeys.ASSET_EVENT_DATE_RANGE]: {
endKey: SearchParamsKeys.END_DATE,
icon: <MdDateRange />,
Expand Down Expand Up @@ -266,6 +276,15 @@ export const useFilterConfigs = () => {
],
type: FilterTypes.SELECT,
},
[SearchParamsKeys.HAS_IMPORT_ERRORS]: {
icon: <MdErrorOutline />,
label: translate("dags:filters.hasImportErrors"),
options: [
{ label: translate("common:yes"), value: "true" },
{ label: translate("common:no"), value: "false" },
],
type: FilterTypes.SELECT,
},
[SearchParamsKeys.HOSTNAME]: {
hotkeyDisabled: true,
icon: <MdComputer />,
Expand Down Expand Up @@ -373,6 +392,12 @@ export const useFilterConfigs = () => {
placeholder: translate("common:table.ownerPlaceholder"),
type: FilterTypes.MULTISELECT,
},
[SearchParamsKeys.PARTITION_KEY]: {
hotkeyDisabled: true,
icon: <MdSearch />,
label: translate("common:dagRun.partitionKey"),
type: FilterTypes.TEXT,
},
[SearchParamsKeys.PARTITION_KEY_PATTERN]: {
hotkeyDisabled: true,
icon: <MdSearch />,
Expand Down Expand Up @@ -490,6 +515,18 @@ export const useFilterConfigs = () => {
})),
type: FilterTypes.SELECT,
},
[SearchParamsKeys.SOURCE_MAP_INDEX]: {
icon: <LuBrackets />,
label: translate("common:mapIndex"),
min: -1,
type: FilterTypes.NUMBER,
},
[SearchParamsKeys.SOURCE_RUN_ID]: {
hotkeyDisabled: true,
icon: <FiBarChart />,
label: translate("common:runId"),
type: FilterTypes.TEXT,
},
[SearchParamsKeys.START_DATE_RANGE]: {
endKey: SearchParamsKeys.START_DATE_LTE,
icon: <MdDateRange />,
Expand Down
5 changes: 5 additions & 0 deletions airflow-core/src/airflow/ui/src/constants/searchParams.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
* under the License.
*/
export enum SearchParamsKeys {
ACTIVE = "active",
ADVANCED_SEARCH = "advanced_search",
AFTER = "after",
ASSET_EVENT_DATE_RANGE = "asset_event_date_range",
Expand Down Expand Up @@ -71,6 +72,7 @@ export enum SearchParamsKeys {
GRAPH_TASK_STATE = "graph-task_state",
GROUP_PATTERN = "group_pattern",
HAS_EVENTS = "has_events",
HAS_IMPORT_ERRORS = "has_import_errors",
HOSTNAME = "hostname",
INCLUDED_EVENTS = "included_events",
ITERATION = "iteration",
Expand Down Expand Up @@ -100,6 +102,7 @@ export enum SearchParamsKeys {
OPERATOR = "operator",
OPERATOR_NAME_PATTERN = "operator_name_pattern",
OWNERS = "owners",
PARTITION_KEY = "partition_key",
PARTITION_KEY_PATTERN = "partition_key_pattern",
PAUSED = "paused",
POOL = "pool",
Expand All @@ -122,6 +125,8 @@ export enum SearchParamsKeys {
SCHEDULING_STATE = "scheduling_state",
SORT = "sort",
SOURCE = "log_source",
SOURCE_MAP_INDEX = "source_map_index",
SOURCE_RUN_ID = "source_run_id",
START_DATE = "start_date",
START_DATE_GTE = "start_date_gte",
START_DATE_LTE = "start_date_lte",
Expand Down
45 changes: 45 additions & 0 deletions airflow-core/src/airflow/ui/src/mocks/handlers/assets.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,53 @@ const plainAsset = {
watchers: [],
};

const assetEvent = {
asset_id: 1,
created_dagruns: [],
extra: {},
group: "asset_group",
id: 100,
name: "asset_with_dependencies",
partition_key: null,
source_dag_id: "producer_dag",
source_map_index: -1,
source_run_id: "manual__2025-01-10",
source_task_id: "producer_task",
timestamp: "2025-01-10T00:00:00Z",
uri: "s3://bucket/asset",
};

// Same asset, but a mapped and partitioned event, so each of the new filters has two rows to
// choose between rather than one row that always matches.
const mappedPartitionedEvent = {
...assetEvent,
id: 101,
partition_key: "2025-01-11",
source_dag_id: "scheduled_producer_dag",
source_map_index: 3,
source_run_id: "scheduled__2025-01-11",
timestamp: "2025-01-11T00:00:00Z",
};

export const assetEvents = [assetEvent, mappedPartitionedEvent];

export const handlers: Array<HttpHandler> = [
http.get("/ui/assets", () => HttpResponse.json({ assets: [assetWithDependencies], total_entries: 1 })),
http.get("/api/v2/assets/events", ({ request }) => {
const url = new URL(request.url);
const sourceRunId = url.searchParams.get("source_run_id");
const sourceMapIndex = url.searchParams.get("source_map_index");
const partitionKey = url.searchParams.get("partition_key");

const filtered = assetEvents.filter(
(event) =>
(sourceRunId === null || event.source_run_id === sourceRunId) &&
(partitionKey === null || event.partition_key === partitionKey) &&
(sourceMapIndex === null || event.source_map_index === Number(sourceMapIndex)),
);

return HttpResponse.json({ asset_events: filtered, total_entries: filtered.length });
}),
http.get("/api/v2/assets/:assetId", ({ params }) =>
HttpResponse.json(params.assetId === "2" ? plainAsset : assetWithDependencies),
),
Expand Down
24 changes: 23 additions & 1 deletion airflow-core/src/airflow/ui/src/mocks/handlers/backfills.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,20 +48,42 @@ const backfillInRange = {
updated_at: "2025-01-16T01:00:00Z",
};

// Still running, so ``completed_at`` is null — the one row the ``active`` filter can single out.
const backfillRunning = {
completed_at: null,
created_at: "2025-02-01T00:00:00Z",
dag_display_name: "tutorial_taskflow_api",
dag_id: "tutorial_taskflow_api",
dag_run_conf: {},
from_date: "2025-02-01T00:00:00Z",
id: 3,
is_paused: false,
max_active_runs: 10,
reprocess_behavior: "none",
to_date: "2025-02-15T00:00:00Z",
updated_at: "2025-02-01T00:00:00Z",
};

export const handlers: Array<HttpHandler> = [
http.get("/ui/backfills", ({ request }) => {
const url = new URL(request.url);
const fromDateGte = url.searchParams.get("from_date_gte");
const fromDateLte = url.searchParams.get("from_date_lte");
const toDateGte = url.searchParams.get("to_date_gte");
const toDateLte = url.searchParams.get("to_date_lte");
const active = url.searchParams.get("active");

const allBackfills = [backfillBeforeFilter, backfillInRange];
const allBackfills = [backfillBeforeFilter, backfillInRange, backfillRunning];

const filtered = allBackfills.filter((backfill) => {
const fromDate = new Date(backfill.from_date);
const toDate = new Date(backfill.to_date);

// The backend models this as "completed_at is null".
if (active !== null && (backfill.completed_at === null) !== (active === "true")) {
return false;
}

if (fromDateGte !== null && fromDate < new Date(fromDateGte)) {
return false;
}
Expand Down
23 changes: 20 additions & 3 deletions airflow-core/src/airflow/ui/src/mocks/handlers/dags.ts
Original file line number Diff line number Diff line change
Expand Up @@ -137,21 +137,37 @@ export const drainingDag = {
timetable_type: "NullTimetable",
};

// Failed to import, so the Dags list has something for the import-errors filter to single out.
export const brokenDag = {
...pausedDag,
dag_display_name: "broken_dag",
dag_id: "broken_dag",
has_import_errors: true,
is_paused: false,
tags: [{ dag_id: "broken_dag", name: "example" }],
};

export const defaultDags = [successDag, failedDag, pausedDag];

const filterDags = ({
dags,
hasImportErrors,
paused,
schedulingState,
timetableTypes,
}: {
dags: Array<(typeof defaultDags)[number] | typeof drainingDag>;
dags: Array<(typeof defaultDags)[number] | typeof brokenDag | typeof drainingDag>;
hasImportErrors: string | null;
paused: string | null;
schedulingState: string | null;
timetableTypes: Array<string>;
}) => {
let allDags = dags;

if (hasImportErrors !== null) {
allDags = allDags.filter((dag) => dag.has_import_errors === (hasImportErrors === "true"));
}

if (timetableTypes.length > 0) {
allDags = allDags.filter((dag) => timetableTypes.includes(dag.timetable_type));
}
Expand All @@ -177,13 +193,14 @@ const filterDags = ({
* enlarging the set every other suite renders.
*/
export const dagsListHandler = (
dags: Array<(typeof defaultDags)[number] | typeof drainingDag> = defaultDags,
dags: Array<(typeof defaultDags)[number] | typeof brokenDag | typeof drainingDag> = defaultDags,
) =>
http.get("/ui/dags", ({ request }) => {
const url = new URL(request.url);
const lastDagRunState = url.searchParams.get("last_dag_run_state");
const anyDagRunState = url.searchParams.get("dag_run_state");
const orderBy = url.searchParams.get("order_by");
const hasImportErrors = url.searchParams.get("has_import_errors");
const paused = url.searchParams.get("paused");
const schedulingState = url.searchParams.get("scheduling_state");
const timetableTypes = url.searchParams.getAll("timetable_type");
Expand All @@ -209,7 +226,7 @@ export const dagsListHandler = (
});
}

let filtered = filterDags({ dags, paused, schedulingState, timetableTypes });
let filtered = filterDags({ dags, hasImportErrors, paused, schedulingState, timetableTypes });

if (orderBy === "last_run_run_after" || orderBy === "-last_run_run_after") {
filtered = [failedDag, successDag, pausedDag, drainingDag].filter((dag) => filtered.includes(dag));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,16 +35,21 @@ export const AssetEvents = () => {
const [sort] = sorting;
const orderBy = sort ? [`${sort.desc ? "-" : ""}${sort.id}`] : ["-timestamp"];

const { DAG_ID, END_DATE, START_DATE, TASK_ID } = SearchParamsKeys;
const { DAG_ID, END_DATE, PARTITION_KEY, SOURCE_MAP_INDEX, SOURCE_RUN_ID, START_DATE, TASK_ID } =
SearchParamsKeys;
const [searchParams] = useSearchParams();
const sourceMapIndex = searchParams.get(SOURCE_MAP_INDEX);

const { data, isLoading } = useAssetServiceGetAssetEvents(
{
assetId,
limit: pagination.pageSize,
offset: pagination.pageIndex * pagination.pageSize,
orderBy,
partitionKey: searchParams.get(PARTITION_KEY) ?? undefined,
sourceDagId: searchParams.get(DAG_ID) ?? undefined,
sourceMapIndex: sourceMapIndex === null || sourceMapIndex === "" ? undefined : Number(sourceMapIndex),
sourceRunId: searchParams.get(SOURCE_RUN_ID) ?? undefined,
sourceTaskId: searchParams.get(TASK_ID) ?? undefined,
timestampGte: searchParams.get(START_DATE) ?? undefined,
timestampLte: searchParams.get(END_DATE) ?? undefined,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
/*!
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import "@testing-library/jest-dom/vitest";
import { render, screen, waitFor } from "@testing-library/react";
import { describe, expect, it } from "vitest";

import { AppWrapper } from "src/utils/AppWrapper";

// The assets mock handler serves two events for asset 1:
// id 100 — producer_dag, run manual__2025-01-10, map index -1, no partition key
// id 101 — scheduled_producer_dag, run scheduled__2025-01-11, map index 3, partition key 2025-01-11
// The card shows the source Dag id rather than the run id, so assertions use the Dag id.
const route = (query: string) => `/assets/1?${query}`;

describe("Asset event filters", () => {
it.each([
{ kept: "producer_dag", query: "source_run_id=manual__2025-01-10" },
{ kept: "scheduled_producer_dag", query: "source_map_index=3" },
{ kept: "scheduled_producer_dag", query: "partition_key=2025-01-11" },
])("narrows the events to one row for $query", async ({ kept, query }) => {
render(<AppWrapper initialEntries={[route(query)]} />);

await waitFor(() => expect(screen.getByText(kept)).toBeInTheDocument());
});

it("shows both events when no filter is applied", async () => {
render(<AppWrapper initialEntries={["/assets/1"]} />);

await waitFor(() => expect(screen.getByText("producer_dag")).toBeInTheDocument());
expect(screen.getByText("scheduled_producer_dag")).toBeInTheDocument();
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ import { BackfillDagRunsModal } from "./BackfillDagRunsModal";
import { BackfillsFilters } from "./BackfillsFilters";

const {
ACTIVE: ACTIVE_PARAM,
COMPLETED_AT_GTE: COMPLETED_AT_GTE_PARAM,
COMPLETED_AT_LTE: COMPLETED_AT_LTE_PARAM,
CREATED_AT_GTE: CREATED_AT_GTE_PARAM,
Expand Down Expand Up @@ -228,9 +229,12 @@ export const Backfills = () => {
const durationGte = searchParams.get(DURATION_GTE_PARAM);
const durationLte = searchParams.get(DURATION_LTE_PARAM);
const reprocessBehaviorParam = searchParams.get(REPROCESS_BEHAVIOR_PARAM);
const activeParam = searchParams.get(ACTIVE_PARAM);
const reprocessBehavior = isReprocessBehavior(reprocessBehaviorParam) ? reprocessBehaviorParam : undefined;

const { data, error, isLoading } = useBackfillServiceListBackfillsUi({
// The backend reads this as "completed_at is null", so only an explicit choice sends it.
active: activeParam === null || activeParam === "" ? undefined : activeParam === "true",
completedAtGte: completedAtGte ?? undefined,
completedAtLte: completedAtLte ?? undefined,
createdAtGte: createdAtGte ?? undefined,
Expand Down
Loading
Loading