This section applies to MRS 3.1.2 or later clusters.
You can customize functions to extend SQL statements to meet personalized requirements. These functions are called user-defined functions (UDFs). You can upload and manage UDF JAR files on the Flink web UI and call UDFs when running jobs.
Flink supports the following three types of UDFs, as described in Table 1.
Type | Description |
|---|---|
User-defined Scalar function (UDF) | Supports one or more input parameters and returns a single result value. For details, see UDF Java and SQL Examples. |
User-defined aggregation function (UDAF) | Aggregates multiple records into one value. For details, see UDAF Java and SQL Examples. |
User-defined table-valued function (UDTF) | Supports one or more input parameters and returns multiple rows or columns. For details, see UDTF Java and SQL Examples. |
You have prepared a UDF JAR file whose size does not exceed 200 MB.
package com.xxx.udf;import org.apache.flink.table.functions.ScalarFunction;public class UdfClass_UDF extends ScalarFunction {public int eval(String s) {return s.length();}}
CREATE TEMPORARY FUNCTION udf as 'com.xxx.udf.UdfClass_UDF';CREATE TABLE udfSource (a VARCHAR) WITH ('connector' = 'datagen','rows-per-second'='1');CREATE TABLE udfSink (a VARCHAR,b int) WITH ('connector' = 'print');INSERT INTOudfSinkSELECTa,udf(a)FROMudfSource;
package com.xxx.udf;import org.apache.flink.table.functions.AggregateFunction;public class UdfClass_UDAF {public static class AverageAccumulator {public int sum;}public static class Average extends AggregateFunction<Integer, AverageAccumulator> {public void accumulate(AverageAccumulator acc, Integer value) {acc.sum += value;}@Overridepublic Integer getValue(AverageAccumulator acc) {return acc.sum;}@Overridepublic AverageAccumulator createAccumulator() {return new AverageAccumulator();}}}
CREATE TEMPORARY FUNCTION udaf as 'com.xxx.udf.UdfClass_UDAF$Average';CREATE TABLE udfSource (a int) WITH ('connector' = 'datagen','rows-per-second'='1','fields.a.min'='1','fields.a.max'='3');CREATE TABLE udfSink (b int,c int) WITH ('connector' = 'print');INSERT INTOudfSinkSELECTa,udaf(a)FROMudfSource group by a;
package com.xxx.udf;import org.apache.flink.api.java.tuple.Tuple2;import org.apache.flink.table.functions.TableFunction;public class UdfClass_UDTF extends TableFunction<Tuple2<String, Integer>> {public void eval(String str) {Tuple2<String, Integer> tuple2 = Tuple2.of(str, str.length());collect(tuple2);}}
CREATE TEMPORARY FUNCTION udtf as 'com.xxx.udf.UdfClass_UDTF';CREATE TABLE udfSource (a VARCHAR) WITH ('connector' = 'datagen','rows-per-second'='1');CREATE TABLE udfSink (b VARCHAR,c int) WITH ('connector' = 'print');INSERT INTOudfSinkSELECTstr,strLengthFROMudfSource,lateral table(udtf(udfSource.a)) as T(str,strLength);