Я пытаюсь использовать зажигание поверх искры и запускаю несколько тестов, чтобы узнать, полезно ли это для нашего конкретного случая использования. Возможность запустить узел Ignite, используя приведенный ниже код. Любая помощь горячо приветствуется. Этот код отлично работает на моем локальном компьютере. Ошибка при работе на Spark cluser. Также весь этот код выполняется с использованием Zeppelin.
import org.apache.ignite.{Ignite, IgniteCache, Ignition}
import org.apache.ignite.configuration.CacheConfiguration
import org.apache.ignite.spark.IgniteDataFrameSettings._
import org.apache.spark.sql.{DataFrame, SaveMode, SparkSession}
val configPath = "/mnt/yarn/data/example-ignite.xml"
val cacheName = "KiCache"
val ignite = Ignition.start(configPath)
val ccfg = new CacheConfiguration[Any, Any](cacheName).setSqlSchema("PUBLIC")
//ccfg.setSqlEscapeAll(true)
val cache = ignite.getOrCreateCache(ccfg)
После этого при попытке сохранить фрейм данных искры как таблицу зажигания возникла ошибка
import org.apache.ignite.Ignite
import org.apache.ignite.spark.IgniteContext
import org.apache.ignite.cache.query.SqlFieldsQuery
import org.apache.ignite.spark.IgniteDataFrameSettings._
import org.apache.spark.sql.{DataFrame, SaveMode, SparkSession}
//val igniteContext = new IgniteContext(sc, configPath)
df.write.format(FORMAT_IGNITE).option(OPTION_CONFIG_FILE,
configPath).option(OPTION_TABLE,
"Test").option(OPTION_CREATE_TABLE_PRIMARY_KEY_FIELDS,
"mtAccountNumber").option(OPTION_CREATE_TABLE_PARAMETERS,
"template=replicated").save()
Ошибка:
org.apache.ignite.IgniteCheckedException: Failed to create Ignite component (consider adding ignite-spring module to classpath) [component=SPRING, cls=org.apache.ignite.internal.util.spring.IgniteSpringHelperImpl]
at org.apache.ignite.internal.IgniteComponentType.componentException(IgniteComponentType.java:320)
at org.apache.ignite.internal.IgniteComponentType.create0(IgniteComponentType.java:296)
at org.apache.ignite.internal.IgniteComponentType.create(IgniteComponentType.java:207)
at org.apache.ignite.internal.IgnitionEx.loadConfigurations(IgnitionEx.java:742)
at org.apache.ignite.internal.IgnitionEx.loadConfigurations(IgnitionEx.java:783)
at org.apache.ignite.internal.IgnitionEx.loadConfiguration(IgnitionEx.java:823)
at org.apache.ignite.spark.impl.IgniteRelationProvider$$anonfun$configProvider$1$1.apply(IgniteRelationProvider.scala:216)
at org.apache.ignite.spark.impl.IgniteRelationProvider$$anonfun$configProvider$1$1.apply(IgniteRelationProvider.scala:213)
at org.apache.ignite.spark.Once.apply(IgniteContext.scala:222)
at org.apache.ignite.spark.IgniteContext.ignite(IgniteContext.scala:144)
at org.apache.ignite.spark.IgniteContext.<init>(IgniteContext.scala:63)
at org.apache.ignite.spark.IgniteContext$.apply(IgniteContext.scala:192)
at org.apache.ignite.spark.impl.IgniteRelationProvider.igniteContext(IgniteRelationProvider.scala:236)
at org.apache.ignite.spark.impl.IgniteRelationProvider.createRelation(IgniteRelationProvider.scala:100)
at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:45)
at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:70)
at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:68)
at org.apache.spark.sql.execution.command.ExecutedCommandExec.doExecute(commands.scala:86)
at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:131)
at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:127)
at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:155)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:152)
at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:127)
at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:80)
at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:80)
at org.apache.spark.sql.DataFrameWriter$$anonfun$runCommand$1.apply(DataFrameWriter.scala:654)
at org.apache.spark.sql.DataFrameWriter$$anonfun$runCommand$1.apply(DataFrameWriter.scala:654)
at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:77)
at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:654)
at org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:273)
at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:267)
... 51 elided
Caused by: java.lang.NoClassDefFoundError: org/springframework/beans/factory/ListableBeanFactory
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:264)
at org.apache.ignite.internal.IgniteComponentType.create0(IgniteComponentType.java:282)
... 81 more
Caused by: java.lang.ClassNotFoundException: org.springframework.beans.factory.ListableBeanFactory
at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:338)
at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
... 84 more