在使用spark streaming+kafka的过程中,通过spark streaming 计算完成后,在使用kafka 将计算结果发送到其他地方进行处理,出现错误。
parseResultJavaDStream.foreachRDD(new VoidFunction<JavaRDD<ParseResult>>() { @Override public void call(JavaRDD<ParseResult> parseResultJavaRDD) throws Exception { parseResultJavaRDD.foreachPartition(new VoidFunction<Iterator<ParseResult>>() { @Override public void call(Iterator<ParseResult> parseResultIterator) throws Exception { Properties properties = new Properties(); properties.setProperty("zookeeper.connect", getWorkItemInfo().getMapValue("pcap.receive.handler.zk.connect")); properties.setProperty("metadata.broker.list", getWorkItemInfo().getMapValue("pcap.receive.handler.data.broker.list")); properties.setProperty("serializer.class", getWorkItemInfo().getMapValue("pcap.receive.handle.serializer.class")); properties.setProperty("key.serializer.class", getWorkItemInfo().getMapValue("pcap.receive.handle.key.serializer.class")); properties.setProperty("request.required.acks", getWorkItemInfo().getMapValue("pcap.receive.handle.request.required.acks")); properties.setProperty("producer.type", getWorkItemInfo().getMapValue("pcap.receive.handle.producer.type")); String m_topicName = getWorkItemInfo().getMapValue("pcap.receive.handle.topicName"); MyProducer myProducer = new MyProducer(properties, m_topicName); while (parseResultIterator.hasNext()) { ParseResult parseResult = parseResultIterator.next(); String protocol = null; String srcAddr = null; String dstAddr = null; int srcPort = -1; int dstPort = -1; int seqNo = -1; boolean isFin = false; for (Item item : parseResult.getItems()) { switch (item.getStepName()) { case "Ethernet": break; case "IPv4": srcAddr = item.getResult().get("srcAddr").toString(); dstAddr = item.getResult().get("dstAddr").toString(); protocol = item.getResult().get("prototype").toString(); break; case "IP": srcPort = item.getResult().get("tcp_srcPort") == null ? 0 : Integer.parseInt(item.getResult().get("udp_srcPort").toString()); dstPort = item.getResult().get("tcp_dstPort") == null ? 0 : Integer.parseInt(item.getResult().get("udp_srcPort").toString()); seqNo = (int) item.getResult().get("sequenceNumber"); isFin = (boolean) item.getResult().get("isFin"); break; case "UDP": srcPort = item.getResult().get("udp_srcPort") == null ? 0 : Integer.parseInt(item.getResult().get("udp_srcPort").toString()); dstPort = item.getResult().get("udp_srcPort") == null ? 0 : Integer.parseInt(item.getResult().get("udp_srcPort").toString()); break; } } PacketMessage newPacket = new PacketMessage(srcAddr, dstAddr, srcPort, dstPort, protocol, seqNo, isFin); newPacket.setPayloadOffset(parseResult.getPayloadOffset()); newPacket.setPayloadLength(parseResult.getPayloadLength()); KeyedMessage<String, PacketMessage> keyedMessage = new KeyedMessage<String, PacketMessage>(m_topicName, String.valueOf(newPacket.getIpTuper().hashCode()), newPacket); myProducer.SendMessage(keyedMessage); logger.error(getWorkItemInfo().getMapValue("pcap.receive.handler.zk.connect")); logger.error(getWorkItemInfo().getMapValue("pcap.receive.handler.data.broker.list")); logger.error(getWorkItemInfo().getMapValue("pcap.receive.handle.serializer.class")); logger.error(getWorkItemInfo().getMapValue("pcap.receive.handle.topicName")); logger.error(String.format("包载荷的起始位置:%s,包载荷的长度:%s", parseResult.getPayloadOffset(), parseResult.getPayloadLength())); logger.error(newPacket.getIpTuper().getProtoType()); } } }); } });
请各位大神帮忙看一下