首页 新闻 会员 周边

kafka BufferUnderFlowExcception

0
悬赏园豆:5 [待解决问题]

在使用spark streaming+kafka的过程中,通过spark streaming 计算完成后,在使用kafka 将计算结果发送到其他地方进行处理,出现错误。

   parseResultJavaDStream.foreachRDD(new VoidFunction<JavaRDD<ParseResult>>() {
            @Override
            public void call(JavaRDD<ParseResult> parseResultJavaRDD) throws Exception {
                parseResultJavaRDD.foreachPartition(new VoidFunction<Iterator<ParseResult>>() {
                    @Override
                    public void call(Iterator<ParseResult> parseResultIterator) throws Exception {
                        Properties properties = new Properties();
                        properties.setProperty("zookeeper.connect", getWorkItemInfo().getMapValue("pcap.receive.handler.zk.connect"));
                        properties.setProperty("metadata.broker.list", getWorkItemInfo().getMapValue("pcap.receive.handler.data.broker.list"));
                        properties.setProperty("serializer.class", getWorkItemInfo().getMapValue("pcap.receive.handle.serializer.class"));
                        properties.setProperty("key.serializer.class", getWorkItemInfo().getMapValue("pcap.receive.handle.key.serializer.class"));
                        properties.setProperty("request.required.acks", getWorkItemInfo().getMapValue("pcap.receive.handle.request.required.acks"));
                        properties.setProperty("producer.type", getWorkItemInfo().getMapValue("pcap.receive.handle.producer.type"));
                        String m_topicName = getWorkItemInfo().getMapValue("pcap.receive.handle.topicName");
                        MyProducer myProducer = new MyProducer(properties, m_topicName);
                        while (parseResultIterator.hasNext()) {
                            ParseResult parseResult = parseResultIterator.next();
                            String protocol = null;
                            String srcAddr = null;
                            String dstAddr = null;
                            int srcPort = -1;
                            int dstPort = -1;
                            int seqNo = -1;
                            boolean isFin = false;
                            for (Item item : parseResult.getItems()) {
                                switch (item.getStepName()) {
                                    case "Ethernet":
                                        break;
                                    case "IPv4":
                                        srcAddr = item.getResult().get("srcAddr").toString();
                                        dstAddr = item.getResult().get("dstAddr").toString();
                                        protocol = item.getResult().get("prototype").toString();
                                        break;
                                    case "IP":
                                        srcPort = item.getResult().get("tcp_srcPort") == null ? 0 : Integer.parseInt(item.getResult().get("udp_srcPort").toString());
                                        dstPort = item.getResult().get("tcp_dstPort") == null ? 0 : Integer.parseInt(item.getResult().get("udp_srcPort").toString());
                                        seqNo = (int) item.getResult().get("sequenceNumber");
                                        isFin = (boolean) item.getResult().get("isFin");
                                        break;
                                    case "UDP":
                                        srcPort = item.getResult().get("udp_srcPort") == null ? 0 : Integer.parseInt(item.getResult().get("udp_srcPort").toString());
                                        dstPort = item.getResult().get("udp_srcPort") == null ? 0 : Integer.parseInt(item.getResult().get("udp_srcPort").toString());
                                        break;
                                }
                            }
                            PacketMessage newPacket = new PacketMessage(srcAddr, dstAddr, srcPort, dstPort, protocol, seqNo, isFin);
                            newPacket.setPayloadOffset(parseResult.getPayloadOffset());
                            newPacket.setPayloadLength(parseResult.getPayloadLength());
                            KeyedMessage<String, PacketMessage> keyedMessage = new KeyedMessage<String, PacketMessage>(m_topicName, String.valueOf(newPacket.getIpTuper().hashCode()), newPacket);
                            myProducer.SendMessage(keyedMessage);
                            logger.error(getWorkItemInfo().getMapValue("pcap.receive.handler.zk.connect"));
                            logger.error(getWorkItemInfo().getMapValue("pcap.receive.handler.data.broker.list"));
                            logger.error(getWorkItemInfo().getMapValue("pcap.receive.handle.serializer.class"));
                            logger.error(getWorkItemInfo().getMapValue("pcap.receive.handle.topicName"));
                            logger.error(String.format("包载荷的起始位置:%s,包载荷的长度:%s", parseResult.getPayloadOffset(), parseResult.getPayloadLength()));
                            logger.error(newPacket.getIpTuper().getProtoType());
                        }
                    }
                });
            }
        });

请各位大神帮忙看一下

CDFanFan的主页 CDFanFan | 初学一级 | 园豆:3
提问于:2016-10-05 10:56
< >
分享
清除回答草稿
   您需要登录以后才能回答,未注册用户请先注册